e.vlasov/redmine_tracker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

оптимизировал сохранение данных и загрузку задач из Redmine

Browse Source
This commit is contained in:
Власов Эдуард
2026-07-06 16:11:41 +03:00
parent 64b347e966
commit 9a0ce15160
5 changed files with 101 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
api/clockify.py
View File

@@ -52,6 +52,7 @@ class ClockifyAPI(ClockifyConfig):
if not tags:
logger.info(f"{Fore.YELLOW}В базе данных нет тегов! Обновите БД при помощи флага -i [--init-db]{Fore.RESET}")
return []
tags_by_id = {tag.id: tag for tag in tags}
result = []
for activity in tqdm(activities, desc="Парсинг задач"):
@@ -64,13 +65,13 @@ class ClockifyAPI(ClockifyConfig):
time_worked_out_total += time_to_hour((end if end else datetime.utcnow()) - start)
tag = [tag for tag in tags if tag.id == tag_id]
tag = tags_by_id.get(tag_id)
if not tag:
tag = self.get_tags(True, tag_id=tag_id, **{"page-size": self.tags_count})
if isinstance(tag, Tag):
db.insert_tags([tag])
tags.append(tag)
tags_by_id[tag.id] = tag
from config import red
try:
task_id = parse_task_id(tag.title)
@@ -84,7 +85,6 @@ class ClockifyAPI(ClockifyConfig):
logger.error(f"{Fore.RED} Тег не найден: {tag_id}{Fore.RESET}")
continue
else:
tag = tag[0]
task_id = parse_task_id(tag.title)
author = db.get_user_by("clockify_user_id", activity["userId"])

1
api/constants.py
View File

@@ -1,3 +1,4 @@
"""Константы API-клиентов."""
REQUEST_TIMEOUT = 30
REDMINE_ISSUES_LIMIT = 100

10
api/redmine.py
View File

@@ -8,7 +8,7 @@ from config import RedmineConfig, config
from classes import Activity, ActivityType
from enums import ActivityTypes
from exceptions import InvalidToken
from .constants import REQUEST_TIMEOUT
from .constants import REDMINE_ISSUES_LIMIT, REQUEST_TIMEOUT
logger = logging.getLogger(__name__)
@@ -142,11 +142,13 @@ class RedmineAPI(RedmineConfig):
def get_issues(self) -> list[Task]:
"""Получает задачи с сервера Redmine"""
issues = []
for i in range(25):
offset = 0
total_count = None
while total_count is None or offset < total_count:
resp = requests.get(
self.base_url + self.issues,
headers={"X-Redmine-Api-Key": self.token},
params={"limit": 100, "offset": 100 * i, "status_id": "*"},
params={"limit": REDMINE_ISSUES_LIMIT, "offset": offset, "status_id": "*"},
timeout=REQUEST_TIMEOUT,
)
if not resp.ok:
@@ -157,6 +159,8 @@ class RedmineAPI(RedmineConfig):
logger.error("Ошибка RedmineAPI: пустой ответ")
return []
issues.extend(data["issues"])
total_count = data.get("total_count", len(issues))
offset += data.get("limit", REDMINE_ISSUES_LIMIT)
return [Task(**{
"id": issue["id"],
"subject": issue["subject"],

164
db/db.py
View File

@@ -33,8 +33,8 @@ class Database:
def insert_activities(self, activities: list[Activity]):
"""Сохраняет в базу данных активности Clockify"""
for act in activities:
try:
for act in activities:
self.cursor.execute(
"""INSERT INTO activity(
id,
@@ -48,7 +48,18 @@ class Database:
task_id,
project_id,
author_id
) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
description = excluded.description,
time_spent = excluded.time_spent,
date_start = excluded.date_start,
date_end = excluded.date_end,
is_tracked = excluded.is_tracked,
vetro_project_id = excluded.vetro_project_id,
tag_id = excluded.tag_id,
task_id = excluded.task_id,
project_id = excluded.project_id,
author_id = excluded.author_id""",
(
act.id,
act.description,
@@ -63,31 +74,16 @@ class Database:
act.author_id,
)
)
except sq.IntegrityError:
self.cursor.execute(
"""UPDATE activity
SET description = ?, time_spent = ?, date_start = ?, date_end = ?, is_tracked = ?, vetro_project_id = ?, tag_id = ?, task_id = ?, project_id = ?
WHERE id = ?""",
(
act.description,
act.time_spent,
act.date_start,
act.date_end,
act.is_tracked,
act.vetro_project_id,
act.tag_id,
act.task_id,
act.project_id,
act.id,
)
)
finally:
except sq.Error as e:
self.con.rollback()
logger.error(f"Не удалось сохранить активности: {e}")
else:
self.con.commit()
def insert_time_entries(self, activities: list[Activity]):
"""Сохраняет в базу данных трудочасы Redmine"""
for act in activities:
try:
for act in activities:
self.cursor.execute(
"""INSERT INTO time_entry
(date, description, activity_id, time_spent, task_id, activity_type_id, author_id)
@@ -102,25 +98,26 @@ class Database:
act.author_id
)
)
self.con.commit()
except sq.IntegrityError as e:
self.con.rollback()
logger.error(f"Не удалось сохранить трудочасы по активности {act.id}: {e}")
logger.error(f"Не удалось сохранить трудочасы: {e}")
else:
self.con.commit()
def insert_tags(self, tags: list[Tag]):
"""Сохраняет в базу данных теги Clockify"""
for tag in tags:
try:
self.cursor.execute("INSERT INTO tag VALUES(?, ?)",(tag.id, tag.title))
except sq.IntegrityError:
for tag in tags:
self.cursor.execute(
"""UPDATE tag
SET title = ?
WHERE id = ?""",
(tag.title, tag.id)
"""INSERT INTO tag VALUES(?, ?)
ON CONFLICT(id) DO UPDATE SET title = excluded.title""",
(tag.id, tag.title)
)
finally:
except sq.Error as e:
self.con.rollback()
logger.error(f"Не удалось сохранить теги: {e}")
else:
self.con.commit()
@@ -128,32 +125,32 @@ class Database:
"""Сохраняет в базу данных Проекты (ДО) Ветро Clockify"""
if not vetro_projects:
return
for vetro_project in vetro_projects:
try:
self.cursor.execute("INSERT INTO vetro_project VALUES(?, ?)", (vetro_project.id, vetro_project.title))
except sq.IntegrityError:
self.cursor.execute("""
UPDATE vetro_project
SET title = ?
WHERE id = ?""",
(vetro_project.title, vetro_project.id)
for vetro_project in vetro_projects:
self.cursor.execute(
"""INSERT INTO vetro_project VALUES(?, ?)
ON CONFLICT(id) DO UPDATE SET title = excluded.title""",
(vetro_project.id, vetro_project.title)
)
finally:
except sq.Error as e:
self.con.rollback()
logger.error(f"Не удалось сохранить проекты Ветро: {e}")
else:
self.con.commit()
def insert_activity_types(self, activity_types: list[ActivityType]):
"""Сохраняет в базу данных Деятельности Redmine"""
for activity_type in activity_types:
try:
self.cursor.execute("INSERT INTO activity_type VALUES(?, ?)", (activity_type.id, activity_type.title))
except sq.IntegrityError:
self.cursor.execute("""
UPDATE activity_type
SET title = ?
WHERE id = ?""",
(activity_type.title, activity_type.id)
for activity_type in activity_types:
self.cursor.execute(
"""INSERT INTO activity_type VALUES(?, ?)
ON CONFLICT(id) DO UPDATE SET title = excluded.title""",
(activity_type.id, activity_type.title)
)
finally:
except sq.Error as e:
self.con.rollback()
logger.error(f"Не удалось сохранить типы деятельности: {e}")
else:
self.con.commit()
def insert_user(self, **kwargs):
@@ -162,35 +159,35 @@ class Database:
self.cursor.execute("""INSERT INTO user(
clockify_user_id, clockify_email, clockify_username,
redmine_user_id, redmine_email, first_name, last_name, redmine_username)
VALUES(?, ?, ?, ?, ?, ?, ?, ?)""", [v for k, v in kwargs.items()])
except sq.IntegrityError:
self.cursor.execute("""
UPDATE user
SET clockify_email = ?,
clockify_username = ?,
redmine_email = ?,
first_name = ?,
last_name = ?,
redmine_username = ?
WHERE redmine_user_id = ? AND clockify_user_id = ?""",
VALUES(?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(clockify_user_id) DO UPDATE SET
clockify_email = excluded.clockify_email,
clockify_username = excluded.clockify_username,
redmine_email = excluded.redmine_email,
first_name = excluded.first_name,
last_name = excluded.last_name,
redmine_username = excluded.redmine_username""",
(
kwargs["clockify_user_id"],
kwargs["clockify_email"],
kwargs["clockify_username"],
kwargs["redmine_user_id"],
kwargs["redmine_email"],
kwargs["first_name"],
kwargs["last_name"],
kwargs["redmine_username"],
kwargs["redmine_user_id"],
kwargs["clockify_user_id"]
)
)
finally:
except sq.Error as e:
self.con.rollback()
logger.error(f"Не удалось сохранить пользователя: {e}")
else:
self.con.commit()
def insert_tasks(self, tasks: list[Task]):
"""Сохраняет в БД Задачи из Redmine"""
for task in tasks:
try:
for task in tasks:
self.cursor.execute("""
INSERT INTO task(
"id",
@@ -202,7 +199,15 @@ class Database:
"tracker",
"project"
)
VALUES(?, ?, ?, ?, ?, ?, ?, ?)""",
VALUES(?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
"subject" = excluded."subject",
"description" = excluded."description",
"status" = excluded."status",
"priority" = excluded."priority",
"author" = excluded."author",
"tracker" = excluded."tracker",
"project" = excluded."project" """,
[
task.id,
task.subject,
@@ -213,29 +218,10 @@ class Database:
task.tracker,
task.project,
])
except sq.IntegrityError:
self.cursor.execute("""
UPDATE task
SET "subject" = ?,
"description" = ?,
"status" = ?,
"priority" = ?,
"author" = ?,
"tracker" = ?,
"project" = ?
WHERE "id" = ?""",
(
task.subject,
task.description,
task.status,
task.priority,
task.author,
task.tracker,
task.project,
task.id
)
)
finally:
except sq.Error as e:
self.con.rollback()
logger.error(f"Не удалось сохранить задачи: {e}")
else:
self.con.commit()
def get_user_by(self, by_field: str, value: str):

3
utils.py
View File

@@ -70,7 +70,8 @@ def db_update():
red_tasks = red.get_issues()
if red_tasks:
db_tasks = db.get_tasks()
clock_tasks = [task for task in red_tasks if task.id in [parse_task_id(tag.title) for tag in [*db_tags, *new_tags]]]
clock_task_ids = {parse_task_id(tag.title) for tag in [*db_tags, *new_tags]}
clock_tasks = [task for task in red_tasks if task.id in clock_task_ids]
new_tasks = [task for task in clock_tasks if task not in db_tasks]
if new_tasks: