e.vlasov/redmine_tracker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

добавил систему миграций и связь одной записи Redmine с несколькими активностями Clockify и запретил отправку трудочасов при инициализации базы данных

Browse Source
This commit is contained in:
Власов Эдуард
2026-07-14 15:54:26 +03:00
parent 5dca3ae446
commit 49286d0fcf
14 changed files with 297 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
README.MD
View File

@@ -98,7 +98,11 @@ redmine-tracker
| `-p [--page-size]`| Количество задач для вывода. Выведет последние n записей | `redmine-tracker --page-size 15` |
| `-d [--debug]` | Режим отладки. Не загружает трудочасы в Redmine | `redmine-tracker -d` |
| `--dry-run` | Предпросмотр без сохранения активностей и отправки трудочасов | `redmine-tracker --dry-run` |
| `-i [--init-db]` | Режим инициализации бд.<br>Подгружает с серверов теги, виды деятельности и Проекты (ДО) Ветро| `redmine-tracker -init-db` |
| `-i [--init-db]` | Выполняет миграции и обновляет справочники без отправки трудочасов в Redmine | `redmine-tracker --init-db` |
При запуске с `--init-db` файлы из каталога `migrations` выполняются по имени.
Успешно применённые миграции повторно не запускаются: их имя и время выполнения
сохраняются в таблице `migration`.
Также можно запускать напрямую из корня проекта:
```bash

1
api/constants.py
View File

@@ -3,6 +3,5 @@
from enums import ActivityTypes
REQUEST_TIMEOUT = 30
REDMINE_ISSUES_LIMIT = 100
ACTIVITY_TYPE_BY_VETRO_PROJECT = {}
DEFAULT_ACTIVITY_TYPE = ActivityTypes.DEVELOPMENT

60
api/redmine.py
View File

@@ -11,7 +11,6 @@ from utils import format_hours
from .constants import (
ACTIVITY_TYPE_BY_VETRO_PROJECT,
DEFAULT_ACTIVITY_TYPE,
REDMINE_ISSUES_LIMIT,
REQUEST_TIMEOUT,
)
@@ -60,7 +59,7 @@ class RedmineAPI(RedmineConfig):
if config.is_debug or resp.ok:
activity.is_tracked = True
if on_success:
on_success(activity)
on_success(activity, resp.json()["time_entry"]["id"])
tracked.append(activity)
logger.info(
f"{Fore.GREEN}{self.format_activity_result(activity, is_beginner)}"
@@ -139,11 +138,19 @@ class RedmineAPI(RedmineConfig):
def get_issue(self, issue_id: int) -> Task | None:
"""Получает задачу по её id с сервера Redmine"""
resp = requests.get(
self.base_url + self.issue.format(**{"id": issue_id}),
headers={"X-Redmine-Api-Key": self.token},
timeout=REQUEST_TIMEOUT,
)
try:
resp = requests.get(
self.base_url + self.issue.format(**{"id": issue_id}),
headers={"X-Redmine-Api-Key": self.token},
timeout=REQUEST_TIMEOUT,
)
except requests.RequestException as error:
logger.error(
"Не удалось загрузить задачу #%s из Redmine: %s",
issue_id,
error,
)
return None
if not resp.ok:
logger.error(f"Ошибка RedmineAPI: {resp.text} {resp.status_code}")
return None
@@ -159,38 +166,13 @@ class RedmineAPI(RedmineConfig):
"project": issue["project"]["name"]
})
def get_issues(self) -> list[Task]:
"""Получает задачи с сервера Redmine"""
issues = []
offset = 0
total_count = None
while total_count is None or offset < total_count:
resp = requests.get(
self.base_url + self.issues,
headers={"X-Redmine-Api-Key": self.token},
params={"limit": REDMINE_ISSUES_LIMIT, "offset": offset, "status_id": "*"},
timeout=REQUEST_TIMEOUT,
)
if not resp.ok:
logger.error(f"Ошибка RedmineAPI: {resp.text} {resp.status_code}")
return []
data = resp.json()
if not data:
logger.error("Ошибка RedmineAPI: пустой ответ")
return []
issues.extend(data["issues"])
total_count = data.get("total_count", len(issues))
offset += data.get("limit", REDMINE_ISSUES_LIMIT)
return [Task(**{
"id": issue["id"],
"subject": issue["subject"],
"description": issue["description"],
"status": issue["status"]["name"],
"priority": issue["priority"]["name"],
"author": issue["author"]["name"],
"tracker": issue["tracker"]["name"],
"project": issue["project"]["name"]
}) for issue in issues]
def get_issues(self, issue_ids: set[int]) -> list[Task]:
"""Получает с сервера Redmine только задачи с указанными номерами."""
return [
task
for issue_id in sorted(issue_ids)
if (task := self.get_issue(issue_id))
]
def _check_token(self):
user = self.get_user()

6
config/__init__.py
View File

@@ -22,6 +22,7 @@ class Config(BaseSettings):
ENV_TXT: str = ".env.txt"
DB_NAME: str = "db/RedmineTracker.db"
SQL_PATH: str = "config/RedmineTracker.sql"
MIGRATIONS_PATH: str = "migrations"
DATETIME_FORMAT: str = "%Y-%m-%dT%H:%M:%SZ"
@@ -83,7 +84,10 @@ def configure_argument_parser():
'-i',
'--init-db',
action="store_true",
help='Режим инициализации бд.\nПодгружает с серверов теги, виды деятельности и проекты Ветро'
help=(
'Режим инициализации БД. Выполняет миграции и обновляет справочники '
'без отправки трудочасов в Redmine'
)
)
return parser

89
db/db.py
View File

@@ -1,4 +1,6 @@
import logging
from importlib.util import module_from_spec, spec_from_file_location
from pathlib import Path
import sqlite3 as sq
from colorama import Fore
@@ -31,6 +33,40 @@ class Database:
logger.error("Ошибка при инициализации БД: ", e)
self.con.rollback()
def run_migrations(self, migrations_path: str):
"""Выполняет новые миграции и сохраняет время их применения."""
self.cursor.execute(
"""CREATE TABLE IF NOT EXISTS migration (
name TEXT PRIMARY KEY,
executed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
)"""
)
self.con.commit()
applied = {
row[0]
for row in self.cursor.execute("SELECT name FROM migration").fetchall()
}
for path in sorted(Path(migrations_path).glob("*.py")):
if path.name.startswith("_") or path.name in applied:
continue
try:
spec = spec_from_file_location(f"migration_{path.stem}", path)
if not spec or not spec.loader:
raise ImportError(f"Не удалось загрузить миграцию {path.name}")
module = module_from_spec(spec)
spec.loader.exec_module(module)
module.upgrade(self.con)
self.cursor.execute(
"INSERT INTO migration(name) VALUES(?)",
(path.name,),
)
self.con.commit()
except Exception:
self.con.rollback()
logger.exception("Не удалось выполнить миграцию %s", path.name)
raise
logger.info("Миграция %s успешно выполнена", path.name)
def insert_activities(self, activities: list[Activity]):
"""Сохраняет в базу данных активности Clockify"""
try:
@@ -80,24 +116,42 @@ class Database:
else:
self.con.commit()
def insert_time_entries(self, activities: list[Activity]):
"""Сохраняет в базу данных трудочасы Redmine"""
def insert_time_entry(
self,
activity: Activity,
source_activities: list[Activity],
redmine_time_entry_id: int,
):
"""Сохраняет запись Redmine и связывает с ней активности Clockify."""
try:
for act in activities:
self.cursor.execute(
self.cursor.execute(
"""INSERT INTO time_entry
(date, description, activity_id, time_spent, task_id, activity_type_id, author_id)
VALUES(?,?,?,?,?,?,?)""",
(
act.date_start.date(),
act.description,
act.id,
act.time_spent,
act.task_id,
ActivityTypes.DEVELOPMENT,
act.author_id
)
(id, date, description, time_spent, task_id, activity_type_id, author_id)
VALUES(?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
date = excluded.date,
description = excluded.description,
time_spent = excluded.time_spent,
task_id = excluded.task_id,
activity_type_id = excluded.activity_type_id,
author_id = excluded.author_id""",
(
redmine_time_entry_id,
activity.date_start.date(),
activity.description,
activity.time_spent,
activity.task_id,
ActivityTypes.DEVELOPMENT,
activity.author_id,
)
)
self.cursor.executemany(
"UPDATE activity SET time_entry_id = ? WHERE id = ?",
[
(redmine_time_entry_id, source_activity.id)
for source_activity in source_activities
],
)
except sq.IntegrityError as e:
self.con.rollback()
logger.error(f"Не удалось сохранить трудочасы: {e}")
@@ -306,7 +360,10 @@ class Database:
def has_time_entry(self, activity_id: str) -> bool:
"""Проверяет, есть ли запись о времени для активности"""
try:
result = self.cursor.execute("SELECT * FROM time_entry WHERE activity_id = ?", (activity_id,)).fetchone()
result = self.cursor.execute(
"SELECT time_entry_id FROM activity WHERE id = ? AND time_entry_id IS NOT NULL",
(activity_id,),
).fetchone()
return bool(result)
except Exception as e:
logger.error(e)

2
main.py
View File

@@ -76,7 +76,9 @@ def main():
return
if args.init_db:
db.run_migrations(config.MIGRATIONS_PATH)
db_update(clock, db, red)
return
try:
TrackingService(

1
migrations/__init__.py Normal file
View File

@@ -0,0 +1 @@
"""Миграции локальной базы данных приложения."""

46
migrations/a7f3c9e1_redmine_clockify_relation.py Normal file
View File

@@ -0,0 +1,46 @@
"""Меняет связь Redmine и Clockify с один-к-одному на один-ко-многим."""
import sqlite3
def upgrade(connection: sqlite3.Connection):
"""Переносит внешний ключ записи Redmine в активности Clockify."""
connection.execute(
"""ALTER TABLE activity
ADD COLUMN time_entry_id INTEGER REFERENCES time_entry(id)"""
)
connection.execute(
"""UPDATE activity
SET time_entry_id = (
SELECT time_entry.id
FROM time_entry
WHERE time_entry.activity_id = activity.id
LIMIT 1
)"""
)
connection.execute(
"""CREATE TABLE time_entry_new (
id INTEGER PRIMARY KEY,
description VARCHAR NOT NULL,
date DATETIME NOT NULL,
time_spent NUMERIC NOT NULL,
task_id INTEGER NOT NULL,
activity_type_id INTEGER NOT NULL,
author_id INTEGER NOT NULL,
FOREIGN KEY (author_id) REFERENCES user(id),
FOREIGN KEY (task_id) REFERENCES task(id),
FOREIGN KEY (activity_type_id) REFERENCES activity_type(id)
)"""
)
connection.execute(
"""INSERT INTO time_entry_new(
id, description, date, time_spent, task_id, activity_type_id, author_id
)
SELECT id, description, date, time_spent, task_id, activity_type_id, author_id
FROM time_entry"""
)
connection.execute("DROP TABLE time_entry")
connection.execute("ALTER TABLE time_entry_new RENAME TO time_entry")
connection.execute(
"CREATE INDEX activity_time_entry_id_idx ON activity(time_entry_id)"
)

2
pyproject.toml
View File

@@ -24,4 +24,4 @@ redmine-tracker = "main:main"
py-modules = ["main", "track", "utils", "enums", "exceptions"]
[tool.setuptools.packages.find]
include = ["api", "classes", "config", "db", "services"]
include = ["api", "classes", "config", "db", "services", "migrations"]

8
services/tracking.py
View File

@@ -108,13 +108,17 @@ class TrackingService:
}
self.db.insert_user(**clock_user, **red_user)
def _save_tracked_activity(self, activity):
def _save_tracked_activity(self, activity, redmine_time_entry_id: int):
"""Сохраняет активность как успешно перенесенную в Redmine."""
source_activities = list(activity.source_activities or (activity,))
for source_activity in source_activities:
source_activity.is_tracked = True
self.db.insert_activities(source_activities)
self.db.insert_time_entries(source_activities)
self.db.insert_time_entry(
activity,
source_activities,
redmine_time_entry_id,
)
def _log_preview(self, activities, activities_to_track):
"""Выводит план переноса активностей без сохранения и отправки."""

78
tests/test_migrations.py Normal file
View File

@@ -0,0 +1,78 @@
"""Тесты применения миграций локальной базы данных."""
from pathlib import Path
from tempfile import TemporaryDirectory
from unittest import TestCase
from db.db import Database
class MigrationTestCase(TestCase):
"""Проверяет миграцию связи Redmine с активностями Clockify."""
def test_migration_creates_one_to_many_relation_and_runs_once(self):
"""Миграция переносит связь и не выполняется повторно."""
with TemporaryDirectory() as directory:
db_path = Path(directory) / "tracker.db"
db = Database(str(db_path), "config/RedmineTracker.sql")
db.run_migrations("migrations")
db.run_migrations("migrations")
activity_columns = {
row[1]
for row in db.cursor.execute("PRAGMA table_info(activity)").fetchall()
}
time_entry_columns = {
row[1]
for row in db.cursor.execute("PRAGMA table_info(time_entry)").fetchall()
}
migration = db.cursor.execute(
"SELECT name, executed_at FROM migration"
).fetchall()
activity_foreign_keys = {
row[3]: row[2]
for row in db.cursor.execute(
"PRAGMA foreign_key_list(activity)"
).fetchall()
}
self.assertIn("time_entry_id", activity_columns)
self.assertNotIn("activity_id", time_entry_columns)
self.assertEqual(activity_foreign_keys["time_entry_id"], "time_entry")
self.assertEqual(len(migration), 1)
self.assertEqual(
migration[0][0],
"a7f3c9e1_redmine_clockify_relation.py",
)
self.assertIsNotNone(migration[0][1])
db.con.close()
def test_multiple_activities_can_reference_one_time_entry(self):
"""Несколько активностей Clockify могут ссылаться на одну запись Redmine."""
with TemporaryDirectory() as directory:
db_path = Path(directory) / "tracker.db"
db = Database(str(db_path), "config/RedmineTracker.sql")
db.run_migrations("migrations")
db.cursor.execute("PRAGMA foreign_keys = OFF")
db.cursor.execute(
"""INSERT INTO time_entry(
id, description, date, time_spent,
task_id, activity_type_id, author_id
) VALUES(42, 'Работа', '2026-07-14', 1, 51906, 9, 1)"""
)
for activity_id in ("first", "second"):
db.cursor.execute(
"""INSERT INTO activity(
id, description, author_id, time_spent, date_start,
tag_id, time_entry_id
) VALUES(?, 'Работа', 1, 0.5, '2026-07-14', 'tag', 42)""",
(activity_id,),
)
count = db.cursor.execute(
"SELECT COUNT(*) FROM activity WHERE time_entry_id = 42"
).fetchone()[0]
self.assertEqual(count, 2)
db.con.close()

43
tests/test_redmine.py Normal file
View File

@@ -0,0 +1,43 @@
"""Тесты выборочной загрузки задач Redmine."""
from unittest import TestCase
from unittest.mock import Mock, patch
import requests
from api.redmine import RedmineAPI
class RedmineAPITestCase(TestCase):
"""Проверяет запрос только необходимых приложению задач."""
def test_get_issues_requests_only_given_issue_ids(self):
"""Задачи загружаются отдельно и только по номерам тегов Clockify."""
redmine = RedmineAPI.__new__(RedmineAPI)
redmine.get_issue = Mock(side_effect=lambda issue_id: issue_id)
result = redmine.get_issues({60130, 51906})
self.assertEqual(result, [51906, 60130])
self.assertEqual(
[call.args[0] for call in redmine.get_issue.call_args_list],
[51906, 60130],
)
def test_get_issues_skips_unavailable_issue(self):
"""Недоступная задача не мешает загрузить остальные указанные задачи."""
redmine = RedmineAPI.__new__(RedmineAPI)
redmine.get_issue = Mock(side_effect=[51906, None])
self.assertEqual(redmine.get_issues({51906, 60130}), [51906])
@patch("api.redmine.requests.get")
def test_get_issue_returns_none_on_timeout(self, get):
"""Сетевой таймаут одной задачи не прерывает обновление БД."""
get.side_effect = requests.ConnectTimeout("Redmine недоступен")
redmine = RedmineAPI.__new__(RedmineAPI)
redmine.base_url = "https://redmine.example/"
redmine.issue = "issues/{id}.json"
redmine.token = "token"
self.assertIsNone(redmine.get_issue(38243))

8
tests/test_tracking.py
View File

@@ -92,11 +92,15 @@ class TrackingServiceTestCase(TestCase):
is_debug=False,
)
service._save_tracked_activity(aggregated_activity)
service._save_tracked_activity(aggregated_activity, 42)
self.assertTrue(all(activity.is_tracked for activity in activities))
db.insert_activities.assert_called_once_with(activities)
db.insert_time_entries.assert_called_once_with(activities)
db.insert_time_entry.assert_called_once_with(
aggregated_activity,
activities,
42,
)
def test_activities_for_different_tasks_are_not_merged(self):
"""Активности разных задач остаются отдельными записями Redmine."""

17
utils.py
View File

@@ -61,10 +61,10 @@ def db_update(clock, db, red):
try:
# Обновление Тегов
logger.info(f"{Fore.YELLOW}Обновление Тегов...\n{Fore.RESET}")
db_tags = db.get_tags() or []
new_tags = []
clock_tags = clock.get_tags(is_updating=True, **{"page-size": clock.tags_count})
if clock_tags:
db_tags = db.get_tags()
new_tags = [tag for tag in clock_tags if tag not in db_tags]
if new_tags:
@@ -79,13 +79,14 @@ def db_update(clock, db, red):
# Обновление Задач
logger.info(f"{Fore.YELLOW}Обновление Задач...\n{Fore.RESET}")
red_tasks = red.get_issues()
clock_task_ids = {
parse_task_id(tag.title)
for tag in [*db_tags, *new_tags]
}
red_tasks = red.get_issues(clock_task_ids)
if red_tasks:
db_tasks = db.get_tasks()
clock_task_ids = {parse_task_id(tag.title) for tag in [*db_tags, *new_tags]}
clock_tasks = [task for task in red_tasks if task.id in clock_task_ids]
new_tasks = [task for task in clock_tasks if task not in db_tasks]
db_tasks = db.get_tasks() or []
new_tasks = [task for task in red_tasks if task not in db_tasks]
if new_tasks:
try: