redesign data extractor and sepparation in get_task_data.py and tasks.py for thw workflows

2 files changed, 265 insertions, 182 deletions

diff --git a/phabricator_tasks/get_task_data.py b/phabricator_tasks/get_task_data.py
new file mode 100644
index 0000000..52d6f08
--- /dev/null
+++ b/phabricator_tasks/get_task_data.py
@@ -0,0 +1,214 @@
+'''
+get all tasks and project information from vyos.dev
+
+    {
+        "task_id": 6473,
+        "task_name": "bgp: missing completion helper for peer-groups inside a VRF",
+        "task_status": "open",
+        "assigned_user": "PHID-USER-xxxxxxxxxx",
+        "assigned_time": "1718107618",
+        "projects": [
+            {
+                "column_name": "In Progress",
+                "column_id": "PHID-PCOL-y523clr222dw2stokcmm",
+                "project_name": "1.4.1",
+                "project_id": "PHID-PROJ-qakk4yrxprecshrgbehq"
+            }
+        ],
+        "task_open": true
+    }
+
+'''
+
+from phabricator import Phabricator as PhabricatorOriginal
+from phabricator import parse_interfaces
+
+'''
+extend of original Phabricator class to add new interface "project.column.search"
+this can be delete if PR https://github.com/disqus/python-phabricator/pull/71 is merged in the pip package
+'''
+
+import copy
+import json
+import pkgutil
+
+INTERFACES = json.loads(
+    pkgutil.get_data('phabricator', 'interfaces.json')
+    .decode('utf-8'))
+
+INTERFACES['project.column.search'] =  {
+        "description": "Search for Workboard columns.",
+        "params": {
+            "ids": "optional list<int>",
+            "phids": "optional list<phid>",
+            "projects": "optional list<phid>"
+        },
+        "return": "list"
+    }
+
+class Phabricator(PhabricatorOriginal):
+    def __init__(self, **kwargs):
+        kwargs['interface'] = copy.deepcopy(parse_interfaces(INTERFACES))
+        super(Phabricator, self).__init__(self, **kwargs)
+        
+''' end of extend the original Phabricator class'''
+
+def phab_api(token):
+    return Phabricator(host='https://vyos.dev/api/', token=token)
+
+def phab_search(method, constraints=dict(), after=None):
+    results = []
+    while True:
+        response = method(
+            constraints=constraints,
+            after=after
+        )
+        results.extend(response.response['data'])
+        after = response.response['cursor']['after']
+        if after is None:
+            break
+    return results
+
+
+def phab_query(method, after=None):
+    results = []
+    while True:
+        response = method(
+            offset=after
+        )
+        results.extend(response.response['data'])
+        after = response.response['cursor']['after']
+        if after is None:
+            break
+    return results
+
+def get_column_name(columnPHID, workboards):
+    for workboard in workboards:
+        if workboard['phid'] == columnPHID:
+            return workboard['fields']['name']
+    return None
+
+def get_project_default_column(project_id, workboards):
+    for workboard in workboards:
+        if workboard['fields']['project']['phid'] == project_id and workboard['fields']['isDefaultColumn']:
+            return workboard['phid'], workboard['fields']['name']
+    return None, None
+
+
+def close_task(task_id, token):
+    phab = phab_api(token)
+    try:
+        response = phab.maniphest.update(
+            id=task_id,
+            status='resolved'
+        )
+        if response.response['isClosed']:
+            print(f'T{task_id} closed')
+    except Exception as e:
+        print(f'T{task_id} Error: {e}')
+
+
+def unassign_task(task_id, token):
+    phab = phab_api(token)
+    raise NotImplementedError
+
+def get_task_data(token):
+
+    phab = phab_api(token)
+    # get list with all open status namens
+    open_status_list = phab.maniphest.querystatuses().response
+    open_status_list = open_status_list.get('openStatuses', None)
+    if not open_status_list:
+        raise Exception('No open status found')
+
+    tasks = phab_search(phab.maniphest.search, constraints={
+                        'statuses': open_status_list
+                    })
+    
+    # get all projects to translate id to name
+    projects_raw = phab_search(phab.project.search)
+    projects = {}
+    for p in projects_raw:
+        projects[p['phid']] = p['fields']['name']
+
+    workboards = phab_search(phab.project.column.search)
+   
+    # get sub-project hirarchy from proxyPHID in workboards
+    project_hirarchy = {}
+    for workboard in workboards:
+        if workboard['fields']['proxyPHID']:
+            proxy_phid = workboard['fields']['proxyPHID']
+            project_phid = workboard['fields']['project']['phid']
+
+            if project_phid not in project_hirarchy.keys():
+                project_hirarchy[project_phid] = []
+            project_hirarchy[project_phid].append(proxy_phid)
+
+    processed_tasks = []
+    for task in tasks:
+        task_data = {
+            'task_id': task['id'],
+            'task_name': task['fields']['name'],
+            'task_status': task['fields']['status']['value'],
+            'assigned_user': task['fields']['ownerPHID'],
+            'assigned_time': None,
+            'projects': []
+        }
+        if task['fields']['status']['value'] in open_status_list:
+            task_data['task_open'] = True
+        else:
+            task_data['task_open'] = False
+        transactions = phab.maniphest.gettasktransactions(ids=[task['id']])
+
+        # transactionType: reassign to get assigened time
+        if task_data['assigned_user']:
+            for transaction in transactions[str(task['id'])]:
+                if transaction['transactionType'] == 'reassign' and transaction['newValue'] == task['fields']['ownerPHID']:
+                    task_data['assigned_time'] = transaction['dateCreated']
+                    break
+        
+        # transactionType: core:edge
+        # loop reversed from oldest to newest transaction
+        # core:edge transactionType is used if the task is moved to another project but stay in default column
+        # this uses the default column (mostly "Need Triage")
+        task_projects = []
+        for transaction in reversed(transactions[str(task['id'])]):
+            if transaction['transactionType'] == 'core:edge':
+                for oldValue in transaction['oldValue']:
+                    if "PHID-PROJ" in oldValue:
+                        task_projects.remove(oldValue)
+
+                for newValue in transaction['newValue']:
+                    if "PHID-PROJ" in newValue:
+                        task_projects.append(newValue)
+        
+        # transactionType: core:columns
+        # use task_projects items as search indicator 'boardPHID' == project_id
+        # remove project from task_projects if the task is moved from the default column to another column
+        for transaction in transactions[str(task['id'])]:
+            if transaction['transactionType'] == 'core:columns':
+                if transaction['newValue'][0]['boardPHID'] in task_projects:
+                    task_projects.remove(transaction['newValue'][0]['boardPHID'])
+                    task_data['projects'].append({
+                        'column_name': get_column_name(transaction['newValue'][0]['columnPHID'], workboards),
+                        'column_id': transaction['newValue'][0]['columnPHID'],
+                        'project_name': projects[transaction['newValue'][0]['boardPHID']],
+                        'project_id': transaction['newValue'][0]['boardPHID'],
+                    })
+
+        
+        # handle remaining projects and set the project base default column
+        for project in task_projects:
+            default_columnid, default_columnname = get_project_default_column(project, workboards)
+            # there are some projects without a workboard like project: "14GA"
+            if default_columnid and default_columnname:
+                task_data['projects'].append({
+                    'column_name': default_columnname,
+                    'column_id': default_columnid,
+                    'project_name': projects[project],
+                    'project_id': project,
+                })
+
+        processed_tasks.append(task_data)
+
+    return processed_tasks
diff --git a/phabricator_tasks/tasks.py b/phabricator_tasks/tasks.py
index c5603b1..bdaa5ce 100644
--- a/phabricator_tasks/tasks.py
+++ b/phabricator_tasks/tasks.py
@@ -1,191 +1,60 @@
-from phabricator import Phabricator as PhabricatorOriginal
-from phabricator import parse_interfaces
-import argparse
-
-
 '''
-get project wide tasks which are not closed but all in the Finished column
 
-1. get all Workboard columns
-    - extract workboard phid for the Finished column
-    - and the project phid and name
+implement Tasks checks and workflows on vyos.dev
 
-2. get all open taks from projects with Finish column
-3. get unique taskslists from previous step to get projekts of a task
-4. get all transactions for each task and check if the task is in the Finished column per project
-5. autoclose if task is in all Finished column
+1. Close a tasks if the Task is in all "Finished" columns
 
-'''
 
 '''
-extend of original Phabricator class to add new interface "project.column.search"
-this can be delete if PR https://github.com/disqus/python-phabricator/pull/71 is merged in the pip package
 
-'''
-import copy
+import argparse
 import json
-import pkgutil
-
-INTERFACES = json.loads(
-    pkgutil.get_data('phabricator', 'interfaces.json')
-    .decode('utf-8'))
-
-INTERFACES['project.column.search'] =  {
-        "description": "Search for Workboard columns.",
-        "params": {
-            "ids": "optional list<int>",
-            "phids": "optional list<phid>",
-            "projects": "optional list<phid>"
-        },
-        "return": "list"
-    }
-
-class Phabricator(PhabricatorOriginal):
-    def __init__(self, **kwargs):
-        kwargs['interface'] = copy.deepcopy(parse_interfaces(INTERFACES))
-        super(Phabricator, self).__init__(self, **kwargs)
-        
-''' end of extend the original Phabricator class'''
-
-def phab_search(method, constraints=dict(), after=None):
-    results = []
-    while True:
-        response = method(
-            constraints=constraints,
-            after=after
-        )
-        results.extend(response.response['data'])
-        after = response.response['cursor']['after']
-        if after is None:
-            break
-    return results
-
-
-def phab_query(method, after=None):
-    results = []
-    while True:
-        response = method(
-            offset=after
-        )
-        results.extend(response.response['data'])
-        after = response.response['cursor']['after']
-        if after is None:
-            break
-    return results
-
-
-def close_task(task_id, phab):
-    try:
-        response = phab.maniphest.update(
-            id=task_id,
-            status='resolved'
-        )
-        if response.response['isClosed']:
-            print(f'T{task_id} closed')
-    except Exception as e:
-        print(f'T{task_id} Error: {e}')
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("-t", "--token", type=str, help="API token", required=True)
-    parser.add_argument("-d", "--dry", help="dry run", action="store_true", default=False)
-    args = parser.parse_args()
-
-    if args.dry:
-        print("This is a dry run")
-
-    phab = Phabricator(host='https://vyos.dev/api/', token=args.token)
-
-    workboards = phab_search(phab.project.column.search)
-    project_hirarchy = {}
-
-    # get sub-project hirarchy from proxyPHID in workboards
-    for workboard in workboards:
-        if workboard['fields']['proxyPHID']:
-            proxy_phid = workboard['fields']['proxyPHID']
-            project_phid = workboard['fields']['project']['phid']
-
-            if project_phid not in project_hirarchy.keys():
-                project_hirarchy[project_phid] = []
-            project_hirarchy[project_phid].append(proxy_phid)
-
-    finished_boards = []
-
-
-    for workboard in workboards:
-        project_id = workboard['fields']['project']['phid']
-        if project_id in project_hirarchy.keys():
-            # skip projects with sub-projects
-            continue
-        if workboard['fields']['name'] == 'Finished':
-            project_tasks = phab_search(phab.maniphest.search, constraints={
-                        'projects': [project_id],
-                        'statuses': ['open'],
-            })
-            finished_boards.append({
-                'project_id': project_id,
-                'project_name': workboard['fields']['project']['name'],
-                'project_tasks': project_tasks,
-                'should_board_id': workboard['phid'],    
-            })
-
-    # get unique tasks
-    # tasks = {
-    #     9999: {
-    #         'PHID-PROJ-xxxxx': 'PHID-PCOL-xxxxx',
-    #         'PHID-PROJ-yyyyy': 'PHID-PCOL-yyyyy'
-    #     }
-    # }
-    tasks = {}
-    for project in finished_boards:
-        project_id = project['project_id']
-        board_id = project['should_board_id']
-        for task in project['project_tasks']:
-            task_id = task['id']
-            if task_id not in tasks.keys():
-                tasks[task_id] = {}
-            if project_id not in tasks[task_id].keys():
-                tasks[task_id][project_id] = board_id
-
-    tasks = dict(sorted(tasks.items()))
-
-    # get transactions for each task and compare if the task is in the Finished column
-    for task_id, projects in tasks.items():
-        project_ids = list(projects.keys())
-        # don't use own pagination function, because endpoint without pagination
-        transactions = phab.maniphest.gettasktransactions(ids=[task_id])
-        transactions = transactions.response[str(task_id)]
-        
-        finished = {}
-        for p in project_ids:
-            finished[p] = False
-        for transaction in transactions:
-            if transaction['transactionType'] == 'core:columns':
-                # test if projectid is in transaction
-                if transaction['newValue'][0]['boardPHID'] in project_ids:
-                    # remove project_id from project_ids to use only last transaction from this project
-                    project_ids.remove(transaction['newValue'][0]['boardPHID'])
-                    # test if boardid is the "Finished" board of project
-                    if projects[transaction['newValue'][0]['boardPHID']] == transaction['newValue'][0]['columnPHID']:
-                        finished[transaction['newValue'][0]['boardPHID']] = True
-
-            # if all core:columns typy of each project_ids is handled.
-            # deside to close task or not
-            if len(project_ids) == 0:
-                if task_id == 6211:
-                    pass
-                task_finish = True
-                for project_id, is_finished in finished.items():
-                    if not is_finished:
-                        task_finish = False
-                if task_finish:
-                    print(f'T{task_id} is Finished in all projects')
-                    if not args.dry:
-                        close_task(task_id, phab)
-                    else:
-                        print(f'T{task_id} would be closed')
+from phabricator_tasks.get_task_data import get_task_data, close_task, unassign_task
+from datetime import datetime, timedelta
+
+parser = argparse.ArgumentParser()
+parser.add_argument("-t", "--token", type=str, help="API token", required=True)
+parser.add_argument("-d", "--dry", help="dry run", action="store_true", default=False)
+args = parser.parse_args()
+
+
+TOKEN = args.token
+DRYRUN = args.dry
+# DRYRUN = True
+UMASIGN_AFTER_DAYS = 90
+UMASIGN_AFTER_DAYS = timedelta(days=UMASIGN_AFTER_DAYS)
+NOW = datetime.now()
+
+if DRYRUN:
+    print("This is a dry run")
+
+tasks = get_task_data(TOKEN)
+
+for task in tasks:
+    # close tasks it is in any projects "finished" column
+    if len(task['projects']) > 0:
+        finished = True
+        for project in task['projects']:
+            if project['column_name'] != 'Finished':
+                finished = False
                 break
-
-
-if __name__ == '__main__':
-    main()
\ No newline at end of file
+        if finished:
+            if DRYRUN:
+                print(f'dryrun: T{task["task_id"]} would be closed')
+            else:
+                close_task(task['task_id'], TOKEN)
+            continue
+    
+
+    '''
+    # unassign tasks with no process after UMASIGN_AFTER_DAYS
+    if task['assigned_user'] and task['assigned_time']:
+        delta = NOW - datetime.fromtimestamp(int(task['assigned_time']))
+        if delta > UMASIGN_AFTER_DAYS:
+            if task['task_status'] != 'open':
+                if DRYRUN:
+                    print(f'dryrun: T{task["task_id"]} with status {task['task_status']} would be unassigned after {delta.days} days')
+                else:
+                    unassign_task(task['task_id'], TOKEN)
+                continue
+    '''
\ No newline at end of file