From 9a0ce15160223cc15335f1891c8bfdee541b907c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=AD=D0=B4=D1=83=D0=B0=D1=80=D0=B4?= Date: Mon, 6 Jul 2026 16:11:41 +0300 Subject: [PATCH] =?UTF-8?q?=D0=BE=D0=BF=D1=82=D0=B8=D0=BC=D0=B8=D0=B7?= =?UTF-8?q?=D0=B8=D1=80=D0=BE=D0=B2=D0=B0=D0=BB=20=D1=81=D0=BE=D1=85=D1=80?= =?UTF-8?q?=D0=B0=D0=BD=D0=B5=D0=BD=D0=B8=D0=B5=20=D0=B4=D0=B0=D0=BD=D0=BD?= =?UTF-8?q?=D1=8B=D1=85=20=D0=B8=20=D0=B7=D0=B0=D0=B3=D1=80=D1=83=D0=B7?= =?UTF-8?q?=D0=BA=D1=83=20=D0=B7=D0=B0=D0=B4=D0=B0=D1=87=20=D0=B8=D0=B7=20?= =?UTF-8?q?Redmine?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/clockify.py | 6 +- api/constants.py | 1 + api/redmine.py | 10 ++- db/db.py | 190 ++++++++++++++++++++++------------------------- utils.py | 3 +- 5 files changed, 101 insertions(+), 109 deletions(-) diff --git a/api/clockify.py b/api/clockify.py index e01e8f0..908a3fe 100644 --- a/api/clockify.py +++ b/api/clockify.py @@ -52,6 +52,7 @@ class ClockifyAPI(ClockifyConfig): if not tags: logger.info(f"{Fore.YELLOW}В базе данных нет тегов! Обновите БД при помощи флага -i [--init-db]{Fore.RESET}") return [] + tags_by_id = {tag.id: tag for tag in tags} result = [] for activity in tqdm(activities, desc="Парсинг задач"): @@ -64,13 +65,13 @@ class ClockifyAPI(ClockifyConfig): time_worked_out_total += time_to_hour((end if end else datetime.utcnow()) - start) - tag = [tag for tag in tags if tag.id == tag_id] + tag = tags_by_id.get(tag_id) if not tag: tag = self.get_tags(True, tag_id=tag_id, **{"page-size": self.tags_count}) if isinstance(tag, Tag): db.insert_tags([tag]) - tags.append(tag) + tags_by_id[tag.id] = tag from config import red try: task_id = parse_task_id(tag.title) @@ -84,7 +85,6 @@ class ClockifyAPI(ClockifyConfig): logger.error(f"{Fore.RED} Тег не найден: {tag_id}{Fore.RESET}") continue else: - tag = tag[0] task_id = parse_task_id(tag.title) author = db.get_user_by("clockify_user_id", activity["userId"]) diff --git a/api/constants.py b/api/constants.py index 5752f4b..76497af 100644 --- a/api/constants.py +++ b/api/constants.py @@ -1,3 +1,4 @@ """Константы API-клиентов.""" REQUEST_TIMEOUT = 30 +REDMINE_ISSUES_LIMIT = 100 diff --git a/api/redmine.py b/api/redmine.py index 4d896c9..e0393ea 100644 --- a/api/redmine.py +++ b/api/redmine.py @@ -8,7 +8,7 @@ from config import RedmineConfig, config from classes import Activity, ActivityType from enums import ActivityTypes from exceptions import InvalidToken -from .constants import REQUEST_TIMEOUT +from .constants import REDMINE_ISSUES_LIMIT, REQUEST_TIMEOUT logger = logging.getLogger(__name__) @@ -142,11 +142,13 @@ class RedmineAPI(RedmineConfig): def get_issues(self) -> list[Task]: """Получает задачи с сервера Redmine""" issues = [] - for i in range(25): + offset = 0 + total_count = None + while total_count is None or offset < total_count: resp = requests.get( self.base_url + self.issues, headers={"X-Redmine-Api-Key": self.token}, - params={"limit": 100, "offset": 100 * i, "status_id": "*"}, + params={"limit": REDMINE_ISSUES_LIMIT, "offset": offset, "status_id": "*"}, timeout=REQUEST_TIMEOUT, ) if not resp.ok: @@ -157,6 +159,8 @@ class RedmineAPI(RedmineConfig): logger.error("Ошибка RedmineAPI: пустой ответ") return [] issues.extend(data["issues"]) + total_count = data.get("total_count", len(issues)) + offset += data.get("limit", REDMINE_ISSUES_LIMIT) return [Task(**{ "id": issue["id"], "subject": issue["subject"], diff --git a/db/db.py b/db/db.py index eeda635..e6d816b 100644 --- a/db/db.py +++ b/db/db.py @@ -33,8 +33,8 @@ class Database: def insert_activities(self, activities: list[Activity]): """Сохраняет в базу данных активности Clockify""" - for act in activities: - try: + try: + for act in activities: self.cursor.execute( """INSERT INTO activity( id, @@ -48,7 +48,18 @@ class Database: task_id, project_id, author_id - ) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + ) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + description = excluded.description, + time_spent = excluded.time_spent, + date_start = excluded.date_start, + date_end = excluded.date_end, + is_tracked = excluded.is_tracked, + vetro_project_id = excluded.vetro_project_id, + tag_id = excluded.tag_id, + task_id = excluded.task_id, + project_id = excluded.project_id, + author_id = excluded.author_id""", ( act.id, act.description, @@ -63,31 +74,16 @@ class Database: act.author_id, ) ) - except sq.IntegrityError: - self.cursor.execute( - """UPDATE activity - SET description = ?, time_spent = ?, date_start = ?, date_end = ?, is_tracked = ?, vetro_project_id = ?, tag_id = ?, task_id = ?, project_id = ? - WHERE id = ?""", - ( - act.description, - act.time_spent, - act.date_start, - act.date_end, - act.is_tracked, - act.vetro_project_id, - act.tag_id, - act.task_id, - act.project_id, - act.id, - ) - ) - finally: - self.con.commit() + except sq.Error as e: + self.con.rollback() + logger.error(f"Не удалось сохранить активности: {e}") + else: + self.con.commit() def insert_time_entries(self, activities: list[Activity]): """Сохраняет в базу данных трудочасы Redmine""" - for act in activities: - try: + try: + for act in activities: self.cursor.execute( """INSERT INTO time_entry (date, description, activity_id, time_spent, task_id, activity_type_id, author_id) @@ -102,59 +98,60 @@ class Database: act.author_id ) ) - self.con.commit() - except sq.IntegrityError as e: - self.con.rollback() - logger.error(f"Не удалось сохранить трудочасы по активности {act.id}: {e}") + except sq.IntegrityError as e: + self.con.rollback() + logger.error(f"Не удалось сохранить трудочасы: {e}") + else: + self.con.commit() def insert_tags(self, tags: list[Tag]): """Сохраняет в базу данных теги Clockify""" - for tag in tags: - try: - self.cursor.execute("INSERT INTO tag VALUES(?, ?)",(tag.id, tag.title)) - except sq.IntegrityError: + try: + for tag in tags: self.cursor.execute( - """UPDATE tag - SET title = ? - WHERE id = ?""", - (tag.title, tag.id) + """INSERT INTO tag VALUES(?, ?) + ON CONFLICT(id) DO UPDATE SET title = excluded.title""", + (tag.id, tag.title) ) - finally: - self.con.commit() + except sq.Error as e: + self.con.rollback() + logger.error(f"Не удалось сохранить теги: {e}") + else: + self.con.commit() def insert_vetro_projects(self, vetro_projects: list[VetroProject]): """Сохраняет в базу данных Проекты (ДО) Ветро Clockify""" if not vetro_projects: return - for vetro_project in vetro_projects: - try: - self.cursor.execute("INSERT INTO vetro_project VALUES(?, ?)", (vetro_project.id, vetro_project.title)) - except sq.IntegrityError: - self.cursor.execute(""" - UPDATE vetro_project - SET title = ? - WHERE id = ?""", - (vetro_project.title, vetro_project.id) + try: + for vetro_project in vetro_projects: + self.cursor.execute( + """INSERT INTO vetro_project VALUES(?, ?) + ON CONFLICT(id) DO UPDATE SET title = excluded.title""", + (vetro_project.id, vetro_project.title) ) - finally: - self.con.commit() + except sq.Error as e: + self.con.rollback() + logger.error(f"Не удалось сохранить проекты Ветро: {e}") + else: + self.con.commit() def insert_activity_types(self, activity_types: list[ActivityType]): """Сохраняет в базу данных Деятельности Redmine""" - for activity_type in activity_types: - try: - self.cursor.execute("INSERT INTO activity_type VALUES(?, ?)", (activity_type.id, activity_type.title)) - except sq.IntegrityError: - self.cursor.execute(""" - UPDATE activity_type - SET title = ? - WHERE id = ?""", - (activity_type.title, activity_type.id) + try: + for activity_type in activity_types: + self.cursor.execute( + """INSERT INTO activity_type VALUES(?, ?) + ON CONFLICT(id) DO UPDATE SET title = excluded.title""", + (activity_type.id, activity_type.title) ) - finally: - self.con.commit() + except sq.Error as e: + self.con.rollback() + logger.error(f"Не удалось сохранить типы деятельности: {e}") + else: + self.con.commit() def insert_user(self, **kwargs): """Сохраняет в базу данных пользователя""" @@ -162,35 +159,35 @@ class Database: self.cursor.execute("""INSERT INTO user( clockify_user_id, clockify_email, clockify_username, redmine_user_id, redmine_email, first_name, last_name, redmine_username) - VALUES(?, ?, ?, ?, ?, ?, ?, ?)""", [v for k, v in kwargs.items()]) - except sq.IntegrityError: - self.cursor.execute(""" - UPDATE user - SET clockify_email = ?, - clockify_username = ?, - redmine_email = ?, - first_name = ?, - last_name = ?, - redmine_username = ? - WHERE redmine_user_id = ? AND clockify_user_id = ?""", + VALUES(?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(clockify_user_id) DO UPDATE SET + clockify_email = excluded.clockify_email, + clockify_username = excluded.clockify_username, + redmine_email = excluded.redmine_email, + first_name = excluded.first_name, + last_name = excluded.last_name, + redmine_username = excluded.redmine_username""", ( + kwargs["clockify_user_id"], kwargs["clockify_email"], kwargs["clockify_username"], + kwargs["redmine_user_id"], kwargs["redmine_email"], kwargs["first_name"], kwargs["last_name"], kwargs["redmine_username"], - kwargs["redmine_user_id"], - kwargs["clockify_user_id"] ) ) - finally: + except sq.Error as e: + self.con.rollback() + logger.error(f"Не удалось сохранить пользователя: {e}") + else: self.con.commit() def insert_tasks(self, tasks: list[Task]): """Сохраняет в БД Задачи из Redmine""" - for task in tasks: - try: + try: + for task in tasks: self.cursor.execute(""" INSERT INTO task( "id", @@ -202,7 +199,15 @@ class Database: "tracker", "project" ) - VALUES(?, ?, ?, ?, ?, ?, ?, ?)""", + VALUES(?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + "subject" = excluded."subject", + "description" = excluded."description", + "status" = excluded."status", + "priority" = excluded."priority", + "author" = excluded."author", + "tracker" = excluded."tracker", + "project" = excluded."project" """, [ task.id, task.subject, @@ -213,30 +218,11 @@ class Database: task.tracker, task.project, ]) - except sq.IntegrityError: - self.cursor.execute(""" - UPDATE task - SET "subject" = ?, - "description" = ?, - "status" = ?, - "priority" = ?, - "author" = ?, - "tracker" = ?, - "project" = ? - WHERE "id" = ?""", - ( - task.subject, - task.description, - task.status, - task.priority, - task.author, - task.tracker, - task.project, - task.id - ) - ) - finally: - self.con.commit() + except sq.Error as e: + self.con.rollback() + logger.error(f"Не удалось сохранить задачи: {e}") + else: + self.con.commit() def get_user_by(self, by_field: str, value: str): """Получает пользователя из бд""" diff --git a/utils.py b/utils.py index 6aa9e8e..12e057e 100644 --- a/utils.py +++ b/utils.py @@ -70,7 +70,8 @@ def db_update(): red_tasks = red.get_issues() if red_tasks: db_tasks = db.get_tasks() - clock_tasks = [task for task in red_tasks if task.id in [parse_task_id(tag.title) for tag in [*db_tags, *new_tags]]] + clock_task_ids = {parse_task_id(tag.title) for tag in [*db_tags, *new_tags]} + clock_tasks = [task for task in red_tasks if task.id in clock_task_ids] new_tasks = [task for task in clock_tasks if task not in db_tasks] if new_tasks: