diff --git a/README.MD b/README.MD index cfc7f3d..bdd4506 100644 --- a/README.MD +++ b/README.MD @@ -98,7 +98,11 @@ redmine-tracker | `-p [--page-size]`| Количество задач для вывода. Выведет последние n записей | `redmine-tracker --page-size 15` | | `-d [--debug]` | Режим отладки. Не загружает трудочасы в Redmine | `redmine-tracker -d` | | `--dry-run` | Предпросмотр без сохранения активностей и отправки трудочасов | `redmine-tracker --dry-run` | -| `-i [--init-db]` | Режим инициализации бд.
Подгружает с серверов теги, виды деятельности и Проекты (ДО) Ветро| `redmine-tracker -init-db` | +| `-i [--init-db]` | Выполняет миграции и обновляет справочники без отправки трудочасов в Redmine | `redmine-tracker --init-db` | + +При запуске с `--init-db` файлы из каталога `migrations` выполняются по имени. +Успешно применённые миграции повторно не запускаются: их имя и время выполнения +сохраняются в таблице `migration`. Также можно запускать напрямую из корня проекта: ```bash diff --git a/api/constants.py b/api/constants.py index b852a1d..3014000 100644 --- a/api/constants.py +++ b/api/constants.py @@ -3,6 +3,5 @@ from enums import ActivityTypes REQUEST_TIMEOUT = 30 -REDMINE_ISSUES_LIMIT = 100 ACTIVITY_TYPE_BY_VETRO_PROJECT = {} DEFAULT_ACTIVITY_TYPE = ActivityTypes.DEVELOPMENT diff --git a/api/redmine.py b/api/redmine.py index 8f2084f..6b39e70 100644 --- a/api/redmine.py +++ b/api/redmine.py @@ -11,7 +11,6 @@ from utils import format_hours from .constants import ( ACTIVITY_TYPE_BY_VETRO_PROJECT, DEFAULT_ACTIVITY_TYPE, - REDMINE_ISSUES_LIMIT, REQUEST_TIMEOUT, ) @@ -60,7 +59,7 @@ class RedmineAPI(RedmineConfig): if config.is_debug or resp.ok: activity.is_tracked = True if on_success: - on_success(activity) + on_success(activity, resp.json()["time_entry"]["id"]) tracked.append(activity) logger.info( f"{Fore.GREEN}{self.format_activity_result(activity, is_beginner)}" @@ -139,11 +138,19 @@ class RedmineAPI(RedmineConfig): def get_issue(self, issue_id: int) -> Task | None: """Получает задачу по её id с сервера Redmine""" - resp = requests.get( - self.base_url + self.issue.format(**{"id": issue_id}), - headers={"X-Redmine-Api-Key": self.token}, - timeout=REQUEST_TIMEOUT, - ) + try: + resp = requests.get( + self.base_url + self.issue.format(**{"id": issue_id}), + headers={"X-Redmine-Api-Key": self.token}, + timeout=REQUEST_TIMEOUT, + ) + except requests.RequestException as error: + logger.error( + "Не удалось загрузить задачу #%s из Redmine: %s", + issue_id, + error, + ) + return None if not resp.ok: logger.error(f"Ошибка RedmineAPI: {resp.text} {resp.status_code}") return None @@ -159,38 +166,13 @@ class RedmineAPI(RedmineConfig): "project": issue["project"]["name"] }) - def get_issues(self) -> list[Task]: - """Получает задачи с сервера Redmine""" - issues = [] - offset = 0 - total_count = None - while total_count is None or offset < total_count: - resp = requests.get( - self.base_url + self.issues, - headers={"X-Redmine-Api-Key": self.token}, - params={"limit": REDMINE_ISSUES_LIMIT, "offset": offset, "status_id": "*"}, - timeout=REQUEST_TIMEOUT, - ) - if not resp.ok: - logger.error(f"Ошибка RedmineAPI: {resp.text} {resp.status_code}") - return [] - data = resp.json() - if not data: - logger.error("Ошибка RedmineAPI: пустой ответ") - return [] - issues.extend(data["issues"]) - total_count = data.get("total_count", len(issues)) - offset += data.get("limit", REDMINE_ISSUES_LIMIT) - return [Task(**{ - "id": issue["id"], - "subject": issue["subject"], - "description": issue["description"], - "status": issue["status"]["name"], - "priority": issue["priority"]["name"], - "author": issue["author"]["name"], - "tracker": issue["tracker"]["name"], - "project": issue["project"]["name"] - }) for issue in issues] + def get_issues(self, issue_ids: set[int]) -> list[Task]: + """Получает с сервера Redmine только задачи с указанными номерами.""" + return [ + task + for issue_id in sorted(issue_ids) + if (task := self.get_issue(issue_id)) + ] def _check_token(self): user = self.get_user() diff --git a/config/__init__.py b/config/__init__.py index ee251eb..8c83604 100644 --- a/config/__init__.py +++ b/config/__init__.py @@ -22,6 +22,7 @@ class Config(BaseSettings): ENV_TXT: str = ".env.txt" DB_NAME: str = "db/RedmineTracker.db" SQL_PATH: str = "config/RedmineTracker.sql" + MIGRATIONS_PATH: str = "migrations" DATETIME_FORMAT: str = "%Y-%m-%dT%H:%M:%SZ" @@ -83,7 +84,10 @@ def configure_argument_parser(): '-i', '--init-db', action="store_true", - help='Режим инициализации бд.\nПодгружает с серверов теги, виды деятельности и проекты Ветро' + help=( + 'Режим инициализации БД. Выполняет миграции и обновляет справочники ' + 'без отправки трудочасов в Redmine' + ) ) return parser diff --git a/db/db.py b/db/db.py index e6d816b..ee26bd5 100644 --- a/db/db.py +++ b/db/db.py @@ -1,4 +1,6 @@ import logging +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path import sqlite3 as sq from colorama import Fore @@ -31,6 +33,40 @@ class Database: logger.error("Ошибка при инициализации БД: ", e) self.con.rollback() + def run_migrations(self, migrations_path: str): + """Выполняет новые миграции и сохраняет время их применения.""" + self.cursor.execute( + """CREATE TABLE IF NOT EXISTS migration ( + name TEXT PRIMARY KEY, + executed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP + )""" + ) + self.con.commit() + applied = { + row[0] + for row in self.cursor.execute("SELECT name FROM migration").fetchall() + } + for path in sorted(Path(migrations_path).glob("*.py")): + if path.name.startswith("_") or path.name in applied: + continue + try: + spec = spec_from_file_location(f"migration_{path.stem}", path) + if not spec or not spec.loader: + raise ImportError(f"Не удалось загрузить миграцию {path.name}") + module = module_from_spec(spec) + spec.loader.exec_module(module) + module.upgrade(self.con) + self.cursor.execute( + "INSERT INTO migration(name) VALUES(?)", + (path.name,), + ) + self.con.commit() + except Exception: + self.con.rollback() + logger.exception("Не удалось выполнить миграцию %s", path.name) + raise + logger.info("Миграция %s успешно выполнена", path.name) + def insert_activities(self, activities: list[Activity]): """Сохраняет в базу данных активности Clockify""" try: @@ -80,24 +116,42 @@ class Database: else: self.con.commit() - def insert_time_entries(self, activities: list[Activity]): - """Сохраняет в базу данных трудочасы Redmine""" + def insert_time_entry( + self, + activity: Activity, + source_activities: list[Activity], + redmine_time_entry_id: int, + ): + """Сохраняет запись Redmine и связывает с ней активности Clockify.""" try: - for act in activities: - self.cursor.execute( - """INSERT INTO time_entry - (date, description, activity_id, time_spent, task_id, activity_type_id, author_id) - VALUES(?,?,?,?,?,?,?)""", - ( - act.date_start.date(), - act.description, - act.id, - act.time_spent, - act.task_id, - ActivityTypes.DEVELOPMENT, - act.author_id - ) + self.cursor.execute( + """INSERT INTO time_entry + (id, date, description, time_spent, task_id, activity_type_id, author_id) + VALUES(?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + date = excluded.date, + description = excluded.description, + time_spent = excluded.time_spent, + task_id = excluded.task_id, + activity_type_id = excluded.activity_type_id, + author_id = excluded.author_id""", + ( + redmine_time_entry_id, + activity.date_start.date(), + activity.description, + activity.time_spent, + activity.task_id, + ActivityTypes.DEVELOPMENT, + activity.author_id, ) + ) + self.cursor.executemany( + "UPDATE activity SET time_entry_id = ? WHERE id = ?", + [ + (redmine_time_entry_id, source_activity.id) + for source_activity in source_activities + ], + ) except sq.IntegrityError as e: self.con.rollback() logger.error(f"Не удалось сохранить трудочасы: {e}") @@ -306,7 +360,10 @@ class Database: def has_time_entry(self, activity_id: str) -> bool: """Проверяет, есть ли запись о времени для активности""" try: - result = self.cursor.execute("SELECT * FROM time_entry WHERE activity_id = ?", (activity_id,)).fetchone() + result = self.cursor.execute( + "SELECT time_entry_id FROM activity WHERE id = ? AND time_entry_id IS NOT NULL", + (activity_id,), + ).fetchone() return bool(result) except Exception as e: logger.error(e) diff --git a/main.py b/main.py index 7d7f197..1745538 100644 --- a/main.py +++ b/main.py @@ -76,7 +76,9 @@ def main(): return if args.init_db: + db.run_migrations(config.MIGRATIONS_PATH) db_update(clock, db, red) + return try: TrackingService( diff --git a/migrations/__init__.py b/migrations/__init__.py new file mode 100644 index 0000000..75a7f15 --- /dev/null +++ b/migrations/__init__.py @@ -0,0 +1 @@ +"""Миграции локальной базы данных приложения.""" diff --git a/migrations/a7f3c9e1_redmine_clockify_relation.py b/migrations/a7f3c9e1_redmine_clockify_relation.py new file mode 100644 index 0000000..324eb66 --- /dev/null +++ b/migrations/a7f3c9e1_redmine_clockify_relation.py @@ -0,0 +1,46 @@ +"""Меняет связь Redmine и Clockify с один-к-одному на один-ко-многим.""" + +import sqlite3 + + +def upgrade(connection: sqlite3.Connection): + """Переносит внешний ключ записи Redmine в активности Clockify.""" + connection.execute( + """ALTER TABLE activity + ADD COLUMN time_entry_id INTEGER REFERENCES time_entry(id)""" + ) + connection.execute( + """UPDATE activity + SET time_entry_id = ( + SELECT time_entry.id + FROM time_entry + WHERE time_entry.activity_id = activity.id + LIMIT 1 + )""" + ) + connection.execute( + """CREATE TABLE time_entry_new ( + id INTEGER PRIMARY KEY, + description VARCHAR NOT NULL, + date DATETIME NOT NULL, + time_spent NUMERIC NOT NULL, + task_id INTEGER NOT NULL, + activity_type_id INTEGER NOT NULL, + author_id INTEGER NOT NULL, + FOREIGN KEY (author_id) REFERENCES user(id), + FOREIGN KEY (task_id) REFERENCES task(id), + FOREIGN KEY (activity_type_id) REFERENCES activity_type(id) + )""" + ) + connection.execute( + """INSERT INTO time_entry_new( + id, description, date, time_spent, task_id, activity_type_id, author_id + ) + SELECT id, description, date, time_spent, task_id, activity_type_id, author_id + FROM time_entry""" + ) + connection.execute("DROP TABLE time_entry") + connection.execute("ALTER TABLE time_entry_new RENAME TO time_entry") + connection.execute( + "CREATE INDEX activity_time_entry_id_idx ON activity(time_entry_id)" + ) diff --git a/pyproject.toml b/pyproject.toml index 1804324..00a2bc0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,4 +24,4 @@ redmine-tracker = "main:main" py-modules = ["main", "track", "utils", "enums", "exceptions"] [tool.setuptools.packages.find] -include = ["api", "classes", "config", "db", "services"] +include = ["api", "classes", "config", "db", "services", "migrations"] diff --git a/services/tracking.py b/services/tracking.py index 53292dc..0bd3048 100644 --- a/services/tracking.py +++ b/services/tracking.py @@ -108,13 +108,17 @@ class TrackingService: } self.db.insert_user(**clock_user, **red_user) - def _save_tracked_activity(self, activity): + def _save_tracked_activity(self, activity, redmine_time_entry_id: int): """Сохраняет активность как успешно перенесенную в Redmine.""" source_activities = list(activity.source_activities or (activity,)) for source_activity in source_activities: source_activity.is_tracked = True self.db.insert_activities(source_activities) - self.db.insert_time_entries(source_activities) + self.db.insert_time_entry( + activity, + source_activities, + redmine_time_entry_id, + ) def _log_preview(self, activities, activities_to_track): """Выводит план переноса активностей без сохранения и отправки.""" diff --git a/tests/test_migrations.py b/tests/test_migrations.py new file mode 100644 index 0000000..5ac15ca --- /dev/null +++ b/tests/test_migrations.py @@ -0,0 +1,78 @@ +"""Тесты применения миграций локальной базы данных.""" + +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest import TestCase + +from db.db import Database + + +class MigrationTestCase(TestCase): + """Проверяет миграцию связи Redmine с активностями Clockify.""" + + def test_migration_creates_one_to_many_relation_and_runs_once(self): + """Миграция переносит связь и не выполняется повторно.""" + with TemporaryDirectory() as directory: + db_path = Path(directory) / "tracker.db" + db = Database(str(db_path), "config/RedmineTracker.sql") + + db.run_migrations("migrations") + db.run_migrations("migrations") + + activity_columns = { + row[1] + for row in db.cursor.execute("PRAGMA table_info(activity)").fetchall() + } + time_entry_columns = { + row[1] + for row in db.cursor.execute("PRAGMA table_info(time_entry)").fetchall() + } + migration = db.cursor.execute( + "SELECT name, executed_at FROM migration" + ).fetchall() + activity_foreign_keys = { + row[3]: row[2] + for row in db.cursor.execute( + "PRAGMA foreign_key_list(activity)" + ).fetchall() + } + + self.assertIn("time_entry_id", activity_columns) + self.assertNotIn("activity_id", time_entry_columns) + self.assertEqual(activity_foreign_keys["time_entry_id"], "time_entry") + self.assertEqual(len(migration), 1) + self.assertEqual( + migration[0][0], + "a7f3c9e1_redmine_clockify_relation.py", + ) + self.assertIsNotNone(migration[0][1]) + db.con.close() + + def test_multiple_activities_can_reference_one_time_entry(self): + """Несколько активностей Clockify могут ссылаться на одну запись Redmine.""" + with TemporaryDirectory() as directory: + db_path = Path(directory) / "tracker.db" + db = Database(str(db_path), "config/RedmineTracker.sql") + db.run_migrations("migrations") + db.cursor.execute("PRAGMA foreign_keys = OFF") + db.cursor.execute( + """INSERT INTO time_entry( + id, description, date, time_spent, + task_id, activity_type_id, author_id + ) VALUES(42, 'Работа', '2026-07-14', 1, 51906, 9, 1)""" + ) + for activity_id in ("first", "second"): + db.cursor.execute( + """INSERT INTO activity( + id, description, author_id, time_spent, date_start, + tag_id, time_entry_id + ) VALUES(?, 'Работа', 1, 0.5, '2026-07-14', 'tag', 42)""", + (activity_id,), + ) + + count = db.cursor.execute( + "SELECT COUNT(*) FROM activity WHERE time_entry_id = 42" + ).fetchone()[0] + + self.assertEqual(count, 2) + db.con.close() diff --git a/tests/test_redmine.py b/tests/test_redmine.py new file mode 100644 index 0000000..1ae92e6 --- /dev/null +++ b/tests/test_redmine.py @@ -0,0 +1,43 @@ +"""Тесты выборочной загрузки задач Redmine.""" + +from unittest import TestCase +from unittest.mock import Mock, patch + +import requests + +from api.redmine import RedmineAPI + + +class RedmineAPITestCase(TestCase): + """Проверяет запрос только необходимых приложению задач.""" + + def test_get_issues_requests_only_given_issue_ids(self): + """Задачи загружаются отдельно и только по номерам тегов Clockify.""" + redmine = RedmineAPI.__new__(RedmineAPI) + redmine.get_issue = Mock(side_effect=lambda issue_id: issue_id) + + result = redmine.get_issues({60130, 51906}) + + self.assertEqual(result, [51906, 60130]) + self.assertEqual( + [call.args[0] for call in redmine.get_issue.call_args_list], + [51906, 60130], + ) + + def test_get_issues_skips_unavailable_issue(self): + """Недоступная задача не мешает загрузить остальные указанные задачи.""" + redmine = RedmineAPI.__new__(RedmineAPI) + redmine.get_issue = Mock(side_effect=[51906, None]) + + self.assertEqual(redmine.get_issues({51906, 60130}), [51906]) + + @patch("api.redmine.requests.get") + def test_get_issue_returns_none_on_timeout(self, get): + """Сетевой таймаут одной задачи не прерывает обновление БД.""" + get.side_effect = requests.ConnectTimeout("Redmine недоступен") + redmine = RedmineAPI.__new__(RedmineAPI) + redmine.base_url = "https://redmine.example/" + redmine.issue = "issues/{id}.json" + redmine.token = "token" + + self.assertIsNone(redmine.get_issue(38243)) diff --git a/tests/test_tracking.py b/tests/test_tracking.py index 2b36c58..698eaf3 100644 --- a/tests/test_tracking.py +++ b/tests/test_tracking.py @@ -92,11 +92,15 @@ class TrackingServiceTestCase(TestCase): is_debug=False, ) - service._save_tracked_activity(aggregated_activity) + service._save_tracked_activity(aggregated_activity, 42) self.assertTrue(all(activity.is_tracked for activity in activities)) db.insert_activities.assert_called_once_with(activities) - db.insert_time_entries.assert_called_once_with(activities) + db.insert_time_entry.assert_called_once_with( + aggregated_activity, + activities, + 42, + ) def test_activities_for_different_tasks_are_not_merged(self): """Активности разных задач остаются отдельными записями Redmine.""" diff --git a/utils.py b/utils.py index 3eada41..aaa94dd 100644 --- a/utils.py +++ b/utils.py @@ -61,10 +61,10 @@ def db_update(clock, db, red): try: # Обновление Тегов logger.info(f"{Fore.YELLOW}Обновление Тегов...\n{Fore.RESET}") - + db_tags = db.get_tags() or [] + new_tags = [] clock_tags = clock.get_tags(is_updating=True, **{"page-size": clock.tags_count}) if clock_tags: - db_tags = db.get_tags() new_tags = [tag for tag in clock_tags if tag not in db_tags] if new_tags: @@ -79,13 +79,14 @@ def db_update(clock, db, red): # Обновление Задач logger.info(f"{Fore.YELLOW}Обновление Задач...\n{Fore.RESET}") - - red_tasks = red.get_issues() + clock_task_ids = { + parse_task_id(tag.title) + for tag in [*db_tags, *new_tags] + } + red_tasks = red.get_issues(clock_task_ids) if red_tasks: - db_tasks = db.get_tasks() - clock_task_ids = {parse_task_id(tag.title) for tag in [*db_tags, *new_tags]} - clock_tasks = [task for task in red_tasks if task.id in clock_task_ids] - new_tasks = [task for task in clock_tasks if task not in db_tasks] + db_tasks = db.get_tasks() or [] + new_tasks = [task for task in red_tasks if task not in db_tasks] if new_tasks: try: