оптимизировал сохранение данных и загрузку задач из Redmine

api/clockify.py

@@ -52,6 +52,7 @@ class ClockifyAPI(ClockifyConfig):
         if not tags:
             logger.info(f"{Fore.YELLOW}В базе данных нет тегов! Обновите БД при помощи флага -i [--init-db]{Fore.RESET}")
             return []
+        tags_by_id = {tag.id: tag for tag in tags}
 
         result = []
         for activity in tqdm(activities, desc="Парсинг задач"):
@@ -64,13 +65,13 @@ class ClockifyAPI(ClockifyConfig):
                 time_worked_out_total += time_to_hour((end if end else datetime.utcnow()) - start)
                 
 
-                tag = [tag for tag in tags if tag.id == tag_id]
+                tag = tags_by_id.get(tag_id)
 
                 if not tag:
                     tag = self.get_tags(True, tag_id=tag_id, **{"page-size": self.tags_count})
                     if isinstance(tag, Tag):
                         db.insert_tags([tag])
-                        tags.append(tag)
+                        tags_by_id[tag.id] = tag
                         from config import red
                         try:
                             task_id = parse_task_id(tag.title)
@@ -84,7 +85,6 @@ class ClockifyAPI(ClockifyConfig):
                         logger.error(f"{Fore.RED} Тег не найден: {tag_id}{Fore.RESET}")
                         continue
                 else:
-                    tag = tag[0]
                     task_id = parse_task_id(tag.title)
 
                 author = db.get_user_by("clockify_user_id", activity["userId"]) 

api/constants.py

@@ -1,3 +1,4 @@
 """Константы API-клиентов."""
 
 REQUEST_TIMEOUT = 30
+REDMINE_ISSUES_LIMIT = 100

api/redmine.py

@@ -8,7 +8,7 @@ from config import RedmineConfig, config
 from classes import Activity, ActivityType
 from enums import ActivityTypes
 from exceptions import InvalidToken
-from .constants import REQUEST_TIMEOUT
+from .constants import REDMINE_ISSUES_LIMIT, REQUEST_TIMEOUT
 
 logger = logging.getLogger(__name__)
 
@@ -142,11 +142,13 @@ class RedmineAPI(RedmineConfig):
     def get_issues(self) -> list[Task]:
         """Получает задачи с сервера Redmine"""
         issues = []
-        for i in range(25):
+        offset = 0
+        total_count = None
+        while total_count is None or offset < total_count:
             resp = requests.get(
                 self.base_url + self.issues,
                 headers={"X-Redmine-Api-Key": self.token},
-                params={"limit": 100, "offset": 100 * i, "status_id": "*"},
+                params={"limit": REDMINE_ISSUES_LIMIT, "offset": offset, "status_id": "*"},
                 timeout=REQUEST_TIMEOUT,
             )
             if not resp.ok:
@@ -157,6 +159,8 @@ class RedmineAPI(RedmineConfig):
                 logger.error("Ошибка RedmineAPI: пустой ответ")
                 return []
             issues.extend(data["issues"])
+            total_count = data.get("total_count", len(issues))
+            offset += data.get("limit", REDMINE_ISSUES_LIMIT)
         return [Task(**{
             "id": issue["id"],
             "subject": issue["subject"],

db/db.py

@@ -33,8 +33,8 @@ class Database:
 
     def insert_activities(self, activities: list[Activity]):
         """Сохраняет в базу данных активности Clockify"""
-        for act in activities:
         try:
+            for act in activities:
                 self.cursor.execute(
                     """INSERT INTO activity(
                        id,
@@ -48,7 +48,18 @@ class Database:
                        task_id,
                        project_id,
                        author_id
-                    ) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
+                    ) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                    ON CONFLICT(id) DO UPDATE SET
+                       description = excluded.description,
+                       time_spent = excluded.time_spent,
+                       date_start = excluded.date_start,
+                       date_end = excluded.date_end,
+                       is_tracked = excluded.is_tracked,
+                       vetro_project_id = excluded.vetro_project_id,
+                       tag_id = excluded.tag_id,
+                       task_id = excluded.task_id,
+                       project_id = excluded.project_id,
+                       author_id = excluded.author_id""",
                     (
                         act.id,
                         act.description,
@@ -63,31 +74,16 @@ class Database:
                         act.author_id,
                     )
                 )
-            except sq.IntegrityError:
+        except sq.Error as e:
-                self.cursor.execute(
+            self.con.rollback()
-                    """UPDATE activity
+            logger.error(f"Не удалось сохранить активности: {e}")
-                    SET description = ?, time_spent = ?, date_start = ?, date_end = ?, is_tracked = ?, vetro_project_id = ?, tag_id = ?, task_id = ?, project_id = ?
+        else:
-                    WHERE id = ?""",
-                    (
-                        act.description,
-                        act.time_spent,
-                        act.date_start,
-                        act.date_end,
-                        act.is_tracked,
-                        act.vetro_project_id,
-                        act.tag_id,
-                        act.task_id,
-                        act.project_id,
-                        act.id,
-                    )
-                )
-            finally:
             self.con.commit()
 
     def insert_time_entries(self, activities: list[Activity]):
         """Сохраняет в базу данных трудочасы Redmine"""
-        for act in activities:
         try:
+            for act in activities:
                 self.cursor.execute(
                 """INSERT INTO time_entry 
                     (date, description, activity_id, time_spent, task_id, activity_type_id, author_id) 
@@ -102,25 +98,26 @@ class Database:
                         act.author_id
                     )
                 )
-                self.con.commit()
         except sq.IntegrityError as e:
             self.con.rollback()
-                logger.error(f"Не удалось сохранить трудочасы по активности {act.id}: {e}")
+            logger.error(f"Не удалось сохранить трудочасы: {e}")
+        else:
+            self.con.commit()
 
 
     def insert_tags(self, tags: list[Tag]):
         """Сохраняет в базу данных теги Clockify"""
-        for tag in tags:
         try:
-                self.cursor.execute("INSERT INTO tag VALUES(?, ?)",(tag.id, tag.title))
+            for tag in tags:
-            except sq.IntegrityError:
                 self.cursor.execute(
-                    """UPDATE tag
+                    """INSERT INTO tag VALUES(?, ?)
-                    SET title = ?
+                    ON CONFLICT(id) DO UPDATE SET title = excluded.title""",
-                    WHERE id = ?""",
+                    (tag.id, tag.title)
-                    (tag.title, tag.id)
                 )
-            finally:
+        except sq.Error as e:
+            self.con.rollback()
+            logger.error(f"Не удалось сохранить теги: {e}")
+        else:
             self.con.commit()
 
     
@@ -128,32 +125,32 @@ class Database:
         """Сохраняет в базу данных Проекты (ДО) Ветро Clockify"""
         if not vetro_projects:
             return
-        for vetro_project in vetro_projects:
         try:
-                self.cursor.execute("INSERT INTO vetro_project VALUES(?, ?)", (vetro_project.id, vetro_project.title))
+            for vetro_project in vetro_projects:
-            except sq.IntegrityError:
+                self.cursor.execute(
-                self.cursor.execute("""
+                    """INSERT INTO vetro_project VALUES(?, ?)
-                    UPDATE vetro_project
+                    ON CONFLICT(id) DO UPDATE SET title = excluded.title""",
-                    SET title = ?
+                    (vetro_project.id, vetro_project.title)
-                    WHERE id = ?""",
-                    (vetro_project.title, vetro_project.id)
                 )
-            finally:
+        except sq.Error as e:
+            self.con.rollback()
+            logger.error(f"Не удалось сохранить проекты Ветро: {e}")
+        else:
             self.con.commit()
 
     def insert_activity_types(self, activity_types: list[ActivityType]):
         """Сохраняет в базу данных Деятельности Redmine"""
-        for activity_type in activity_types:
         try:
-                self.cursor.execute("INSERT INTO activity_type VALUES(?, ?)", (activity_type.id, activity_type.title))
+            for activity_type in activity_types:
-            except sq.IntegrityError:
+                self.cursor.execute(
-                self.cursor.execute("""
+                    """INSERT INTO activity_type VALUES(?, ?)
-                    UPDATE activity_type
+                    ON CONFLICT(id) DO UPDATE SET title = excluded.title""",
-                    SET title = ?
+                    (activity_type.id, activity_type.title)
-                    WHERE id = ?""",
-                    (activity_type.title, activity_type.id)
                 )
-            finally:
+        except sq.Error as e:
+            self.con.rollback()
+            logger.error(f"Не удалось сохранить типы деятельности: {e}")
+        else:
             self.con.commit()
 
     def insert_user(self, **kwargs):
@@ -162,35 +159,35 @@ class Database:
             self.cursor.execute("""INSERT INTO user(
                                 clockify_user_id, clockify_email, clockify_username,
                                 redmine_user_id, redmine_email, first_name, last_name, redmine_username)
-                                VALUES(?, ?, ?, ?, ?, ?, ?, ?)""", [v for k, v in kwargs.items()])
+                                VALUES(?, ?, ?, ?, ?, ?, ?, ?)
-        except sq.IntegrityError:
+                                ON CONFLICT(clockify_user_id) DO UPDATE SET
-            self.cursor.execute("""
+                                    clockify_email = excluded.clockify_email,
-                UPDATE user
+                                    clockify_username = excluded.clockify_username,
-                SET clockify_email = ?,
+                                    redmine_email = excluded.redmine_email,
-                    clockify_username = ?,
+                                    first_name = excluded.first_name,
-                    redmine_email = ?,
+                                    last_name = excluded.last_name,
-                    first_name = ?,
+                                    redmine_username = excluded.redmine_username""",
-                    last_name = ?,
-                    redmine_username = ?
-                WHERE redmine_user_id = ? AND clockify_user_id = ?""",
                 (
+                    kwargs["clockify_user_id"],
                     kwargs["clockify_email"],
                     kwargs["clockify_username"],
+                    kwargs["redmine_user_id"],
                     kwargs["redmine_email"],
                     kwargs["first_name"],
                     kwargs["last_name"],
                     kwargs["redmine_username"],
-                    kwargs["redmine_user_id"],
-                    kwargs["clockify_user_id"]
                 )
             )
-        finally:
+        except sq.Error as e:
+            self.con.rollback()
+            logger.error(f"Не удалось сохранить пользователя: {e}")
+        else:
             self.con.commit()
             
     def insert_tasks(self, tasks: list[Task]):
         """Сохраняет в БД Задачи из Redmine"""
-        for task in tasks:
         try:
+            for task in tasks:
                 self.cursor.execute("""
                     INSERT INTO task(
                         "id",
@@ -202,7 +199,15 @@ class Database:
                         "tracker",
                         "project"
                     )
-                    VALUES(?, ?, ?, ?, ?, ?, ?, ?)""",
+                    VALUES(?, ?, ?, ?, ?, ?, ?, ?)
+                    ON CONFLICT(id) DO UPDATE SET
+                        "subject" = excluded."subject",
+                        "description" = excluded."description",
+                        "status" = excluded."status",
+                        "priority" = excluded."priority",
+                        "author" = excluded."author",
+                        "tracker" = excluded."tracker",
+                        "project" = excluded."project" """,
                     [
                         task.id,
                         task.subject,
@@ -213,29 +218,10 @@ class Database:
                         task.tracker,
                         task.project,
                     ])
-            except sq.IntegrityError:
+        except sq.Error as e:
-                self.cursor.execute("""
+            self.con.rollback()
-                    UPDATE task
+            logger.error(f"Не удалось сохранить задачи: {e}")
-                    SET "subject" = ?,
+        else:
-                        "description" = ?,
-                        "status" = ?,
-                        "priority" = ?,
-                        "author" = ?,
-                        "tracker" = ?,
-                        "project" = ?
-                    WHERE "id" = ?""",
-                    (
-                        task.subject,
-                        task.description,
-                        task.status,
-                        task.priority,
-                        task.author,
-                        task.tracker,
-                        task.project,
-                        task.id
-                    )
-                )
-            finally:
             self.con.commit()
     
     def get_user_by(self, by_field: str, value: str):

utils.py

@@ -70,7 +70,8 @@ def db_update():
         red_tasks = red.get_issues()
         if red_tasks:
             db_tasks = db.get_tasks()
-            clock_tasks = [task for task in red_tasks if task.id in [parse_task_id(tag.title) for tag in [*db_tags, *new_tags]]]
+            clock_task_ids = {parse_task_id(tag.title) for tag in [*db_tags, *new_tags]}
+            clock_tasks = [task for task in red_tasks if task.id in clock_task_ids]
             new_tasks = [task for task in clock_tasks if task not in db_tasks]
 
             if new_tasks: