diff --git a/gitlab/gitlab.zip b/gitlab/gitlab.zip index 0817c6a..9a06e60 100644 Binary files a/gitlab/gitlab.zip and b/gitlab/gitlab.zip differ diff --git a/gitlab/mapping.py b/gitlab/mapping.py index 5d0b051..63dfd93 100644 --- a/gitlab/mapping.py +++ b/gitlab/mapping.py @@ -113,6 +113,10 @@ def fetch_fieldtype_info(self): "type": fields_type["DATE"], "system": "due_date_fixed" }, + "start_date":{ + "type":fields_type["DATE"], + "system": "start_date" + }, "due_date":{ "type":fields_type["DATE"], "system": "due_date" @@ -248,6 +252,12 @@ def test_connection(self): class WebHook(BaseWebHook): + @staticmethod + def _extract_group_id(project_info): + # No Project stores organization id as "/No Project". + project_value = str(project_info.get("project", "")) + return project_value.split("/", 1)[0] + def create_webhook(self, webhook_name, webhook_url, webhook_description, project_id): payload = { @@ -258,5 +268,13 @@ def create_webhook(self, webhook_name, webhook_url, webhook_description, project } # Payload data to create single webhook for project in self.projects_info: - transformer_functions.webhooks(self.instance_obj, id=project['project'] - , payload=payload) + if project["id"] != project_id: + continue + + if project["id"] == "no_project": + group_id = self._extract_group_id(project) + transformer_functions.group_webhooks(self.instance_obj, id=group_id, payload=payload) + else: + transformer_functions.webhooks(self.instance_obj, id=project['project'], payload=payload) + + break diff --git a/gitlab/sync.py b/gitlab/sync.py index 2900134..607213c 100644 --- a/gitlab/sync.py +++ b/gitlab/sync.py @@ -18,18 +18,22 @@ from dateutil import parser import time import json +from urllib.parse import quote, parse_qs from external_plugins.gitlab import transformer_functions from requests.exceptions import HTTPError +from agilitysync.lib.plugin_manage import plugin_schema +from agilitysync.external_lib.restapi import ASyncRestApi +from bson import ObjectId class AttachmentUpload(BaseAttachmentUpload): def fetch_url(self): - - return "{}/{}".format("'https://gitlab.com",self.private_data["url"]) + + return "{}/{}".format("'https://gitlab.com",self.private_data["url"]) def fetch_upload_url(self): - - return "{}/{}/{}/{}".format(self.instance_details["url"],"projects",self.outbound.project_info["project"],"uploads") + api_url = transformer_functions.get_api_url(self.instance_details["url"]) + return "{}/{}/{}/{}".format(api_url, "projects", self.outbound.project_info["project"], "uploads") def fetch_multipart_data(self, storage_obj): payload = { "file" : storage_obj @@ -46,15 +50,16 @@ def upload_on_success(self,response): res = [] res.append(eval(response.text)) self.private_data = eval(response.text) + payload = { + "body": "![{}]({})".format(self.filename, res[0]["url"]) + } try: - payload = { - "body": "![{}]({})".format(self.filename,res[0]["url"]) - } transformer_functions.comment(self.instance_object, self.outbound.project_info, payload, self.outbound.workitem_display_id) - except Exception as e: - error_msg = 'Unable to sync comment. Error is [{}]. The comment is [{}]'.format(str(e), comment) + error_msg = 'Unable to sync comment. Error is [{}]. The comment is [{}]'.format( + str(e), payload.get("body", "") + ) raise as_exceptions.OutboundError(error_msg, stack_trace=True) def remove(self, attachment_id, verify_tls): @@ -63,16 +68,130 @@ def remove(self, attachment_id, verify_tls): {'id': attachment_id}, query_str="op=Delete") class AttachmentDownload(BaseAttachmentDownload): def basic_auth(self): - return {"rohithamroser","Pwe_96fww3:PF6h"} + return None class Payload(BasePayload): + def _get_instance_details(self): + if getattr(self, "_payload_instance_details", None) is None: + details = plugin_schema.get_plugin_instance_by_id(self.instance_id, self.db) + if "url" in details: + details["url"] = details["url"].rstrip("/") + self._payload_instance_details = details + return self._payload_instance_details + + def _get_instance_object(self): + if getattr(self, "_payload_instance_object", None) is None: + details = self._get_instance_details() + # Group work-item lookup may run with PAT tokens where PRIVATE-TOKEN auth is required. + self._payload_instance_object = ASyncRestApi(transformer_functions.get_api_url(details['url']), headers={ + "PRIVATE-TOKEN": "{}".format(details["token"]), + "Accept": "application/json", + "Content-Type": "application/json" + }) + return self._payload_instance_object + + def _group_full_path_from_url(self, event): + workitem_url = (event.get("object_attributes") or {}).get("url", "") + # Example: https://gitlab.com/groups/my-group/-/work_items/4 + match = re.search(r"/groups/(?P.+?)/-/work_items/", workitem_url) + return match.group("group_path") if match else None + + def _group_id_from_event(self, event): + group_path = self._group_full_path_from_url(event) + if not group_path: + return None + + # GitLab group paths can contain '/'; API expects URL-encoded full path. + encoded_group = quote(group_path, safe="") + try: + group_info = self._get_instance_object().get("groups/{}".format(encoded_group)) + except Exception: + # Fallback to plugin's standard connector for environments using bearer-style tokens. + group_info = transformer_functions.connect(self._get_instance_details()).get("groups/{}".format(encoded_group)) + return str(group_info.get("id")) if group_info and group_info.get("id") is not None else None + + def _handler_name_from_event(self, event): + if event.get("handler"): + return event.get("handler") + + headers = event.get("__HEADERS") or {} + query = headers.get("query") or headers.get("QUERY_STRING") or "" + if query.startswith("?"): + query = query[1:] + + handler_name = parse_qs(query).get("handler") + return handler_name[0] if handler_name else None + + def _project_from_handler_mapping(self, event): + handler_name = self._handler_name_from_event(event) + if not handler_name: + return None + + webhook_doc = self.db.webhookhandlers.find_one({"name": handler_name}) + if not webhook_doc: + return None + + args = ((webhook_doc.get("directives") or [{}])[0].get("details") or {}).get("args") or {} + map_names = args.get("map_names") or [] + no_project_candidates = [] + + for map_name in map_names: + try: + map_doc = self.db.data_map.find_one({"_id": ObjectId(map_name), "active": True}) + except Exception: + map_doc = None + if not map_doc: + continue + + for direction in ("forward-direction", "backward-direction"): + direction_doc = map_doc.get(direction) or {} + inbound_doc = direction_doc.get("inbound") or {} + if inbound_doc.get("instance_id") != self.instance_id: + continue + + for system_map in direction_doc.get("system_mapping") or []: + project_info = ((system_map.get("inbound_plugin_config") or {}).get("project_info") or {}) + if project_info.get("id") == "no_project" and project_info.get("project"): + no_project_candidates.append(project_info.get("project")) + + # Remove duplicates while preserving order. + unique_candidates = list(dict.fromkeys(no_project_candidates)) + + if len(unique_candidates) == 1: + return unique_candidates[0] + + if len(unique_candidates) > 1: + # If handler has multiple no-project mappings, use event group id to disambiguate. + group_id = self._group_id_from_event(event) + if group_id: + for candidate in unique_candidates: + if str(candidate).split("/", 1)[0] == str(group_id): + return candidate + + return None + def fetch_project(self, event): - project = str(event["project"]["id"]) - org = event['project']['namespace'] - return project + if event.get("project") and event["project"].get("id") is not None: + return str(event["project"]["id"]) + + # Fallback for older GitLab webhook formats where project_id is at root level + if event.get("project_id") is not None: + return str(event["project_id"]) + + mapped_project = self._project_from_handler_mapping(event) + if mapped_project: + return mapped_project + + group_id = self._group_id_from_event(event) + if group_id: + return "{}/No Project".format(group_id) + + raise as_exceptions.PayloadError("Unable to derive project/group id from GitLab payload") def fetch_asset(self, event): + if event.get("object_kind") == "work_item" and (event.get("object_attributes") or {}).get("type") == "Epic": + return "Assettype-002" return "Assettype-001" def is_cyclic_event(self, event, sync_user): @@ -83,25 +202,25 @@ class Event(BaseEvent): def fetch_event_type(self): - if self.event['object_attributes']["updated_at"] == self.event['object_attributes']["created_at"]: - event_type = "open" - if self.event['object_attributes']["updated_at"] != self.event['object_attributes']["created_at"] or self.event["event_type"] == "note" : + if self.event["event_type"] == "note": event_type = "update" + elif self.event['object_attributes']["updated_at"] == self.event['object_attributes']["created_at"]: + event_type = self.event['object_attributes'].get("action", "open") else: - event_type = self.event['object_attributes']["action"] + event_type = self.event['object_attributes'].get("action", "update") - if event_type in ('open'): + if event_type == "open": return EventTypes.CREATE - elif event_type == ('close'): + elif event_type == "close": return EventTypes.DELETE - elif event_type in ('update',): + elif event_type == "update": return EventTypes.UPDATE error_msg = 'Unsupported event type [{}]'.format(event_type) raise as_exceptions.PayloadError(error_msg) def fetch_workitem_id(self): - if self.event["event_type"] == "issue": + if self.event["event_type"] in ("issue", "work_item"): return str(self.event['object_attributes']['id']) else: return str(self.event['issue']['id']) @@ -113,7 +232,7 @@ def fetch_workitem_display_id(self): return self.event['object_attributes']['id'] def fetch_workitem_url(self): - return "/".join(self.event['object_attributes']['url'].split("/")[1:]) + return "/".join(self.event['object_attributes']['url'].split("/")[3:]) def fetch_revision(self): return self.event['object_attributes']["updated_at"] @@ -144,28 +263,50 @@ def is_comment_updated(self, updated_at_with_time, latest_public_comment_html): def fetch_event_category(self): category = [] + object_kind = self.event.get("object_kind") or self.event.get("event_type") or "" - if self.event['object_attributes'].get("action") is not None: - + if object_kind == "note": + note_text = self.event['object_attributes'].get("note") or "" + if "/upload/" in note_text: + category.append(EventCategory.ATTACHMENT) + else: + category.append(EventCategory.COMMENT) + elif object_kind in ("issue", "work_item") or self.event['object_attributes'].get("action") is not None: category.append(EventCategory.WORKITEM) - elif self.event['object_attributes']["note"].find("/upload/") is True: - category.append(EventCategory.ATTACHMENT) else: category.append(EventCategory.COMMENT) - return category def fetch_parent_id(self): parent_id = None old_parent_id = None - _id = transformer_functions.get_parent_id(proj_id=self.event['object_attributes']["project_id"], - iid=self.event['object_attributes']["iid"], + # project_id may be absent from object_attributes in work_item events; + # fall back to root-level project_id or project.id + proj_id = ( + self.event['object_attributes'].get("project_id") + or self.event.get("project_id") + or (self.event.get("project") or {}).get("id") + ) + iid = self.event['object_attributes'].get("iid") + if not proj_id or not iid: + return None, old_parent_id + _id = transformer_functions.get_parent_id(proj_id=proj_id, + iid=iid, instance=self.instance_object) - if _id["epic"] is not None: - parent_id = _id["epic"]["id"] - - return str(parent_id), old_parent_id + if _id and isinstance(_id, dict) and (_id.get("epic") is not None): + epic_info = _id["epic"] + group_id = epic_info.get("group_id") + epic_iid = epic_info.get("iid") + if group_id and epic_iid: + try: + epic_detail = transformer_functions.get_epic( + self.instance_object, {"project": str(group_id)}, epic_iid) + parent_id = epic_detail.get("work_item_id") + except Exception: + parent_id = None + + return parent_id, old_parent_id def normalize_texttype_multivalue_field(self, field_value, field_attr): @@ -195,6 +336,28 @@ def normalize_texttype_multivalue_field(self, field_value, field_attr): return vals def migrate_create(self): + is_epic = self.project_info.get("display_name") == "No Project" + + if is_epic: + res = transformer_functions.get_epic(self.instance_object, self.project_info, self.workitem_display_id) + event = {"object_attributes": res} + labels = [] + for label_title in (res.get("labels") or []): + labels.append({ + "created_at": res["created_at"], + "updated_at": res["updated_at"], + "title": label_title, + }) + event["object_attributes"]["action"] = "open" + event["object_kind"] = "work_item" + event["event_type"] = "work_item" + event["object_attributes"]["type"] = "Epic" + event["object_attributes"]["created_at"] = res["created_at"] + event["object_attributes"]["updated_at"] = res["created_at"] + event["object_attributes"]["labels"] = labels + event["object_attributes"]["url"] = res.get("web_url") or res.get("url", "") + return [event] + res = transformer_functions.get_issue(self.instance_object, self.project_info, self.workitem_display_id) event = { @@ -421,7 +584,9 @@ def create(self, sync_fields): xref_object = { "relative_url": "{}/{}/{}/{}".format("repos", org, self.project_info["display_name"], "issues"), 'id': str(ticket['id']), - 'display_id': str(ticket['iid']), + # For epics, GitLab 15.9+ returns work_item_iid alongside iid. + # Work Items Notes API uses work_item_iid; fall back to iid if not present. + 'display_id': str(ticket.get('work_item_iid') or ticket['iid']), 'sync_info': sync_info, } xref_object["absolute_url"] = xref_object["relative_url"] diff --git a/gitlab/transformer_functions.py b/gitlab/transformer_functions.py index 07543a0..00dd796 100644 --- a/gitlab/transformer_functions.py +++ b/gitlab/transformer_functions.py @@ -130,18 +130,18 @@ "required": False, "disabled field": False, "customfield": False, - "raw_title": "start_date_fixed", - "title": "start_date_fixed", - "type": "start_date_fixed", + "raw_title": "start_date", + "title": "start_date", + "type": "start_date", "IsMultivalue": False }, { "required": False, "disabled field": False, "customfield": False, - "raw_title": "due_date_fixed", - "title": "due_date_fixed", - "type": "due_date_fixed", + "raw_title": "due_date", + "title": "due_date", + "type": "due_date", "IsMultivalue": False }, { @@ -172,31 +172,42 @@ def get_issue(instance, proj_id, workid): return instance.get(path) +def get_epic(instance, proj_id, workid): + group_id = str(proj_id["project"]).split("/")[0] + path = "{}/{}/{}/{}".format("groups", group_id, "epics", workid) + return instance.get(path) + + def epicfields(): return fields_epic +def get_api_url(url): + base_url = str(url).rstrip("/") + if base_url.endswith("/api/v4"): + return base_url + return "{}/api/v4".format(base_url) + + def connect(instance_details): - return ASyncRestApi(instance_details['url'], headers={ - "authorization": "Bearer {}".format(instance_details["token"]), + if not instance_details.get('url'): + raise ValueError("GitLab URL is required. Please provide the domain URL in the instance configuration.") + return ASyncRestApi(get_api_url(instance_details['url']), headers={ + "PRIVATE-TOKEN": "{}".format(instance_details["token"]), "Accept": "application/json", "Content-Type": "application/json" }) def check_connection(instance, instance_details): - path = "{}".format( - DEFAULT.INITIAL_PATH, - instance_details["Username"] - - ) + path = "{}".format(DEFAULT.INITIAL_PATH) response = instance.get(path) - if instance_details["Username"] == response["username"]: + if response.get("username"): return "Connection to Gitlab server is successfull." else: - return response["error"] + raise Exception("GitLab API did not return a valid user. Please verify your token.") def get_org(instance): @@ -365,6 +376,21 @@ def webhooks(instance, id, payload=None): return response +def group_webhooks(instance, id, payload=None): + instance_path = "hooks" + path = "{}/{}/{}".format( + "groups", + id, + instance_path) + + if payload: + response = instance.post(path, payload) + return response + else: + response = instance.get(path) + return response + + def get_fields_values(instance, field): path = "{}/{}" @@ -382,8 +408,8 @@ def get_parent_id(proj_id, iid, instance): try: response = instance.get(path) return response - except: - return response["error"] + except Exception: + return None def multivalue_fetch(instance, proj_id, field): @@ -474,9 +500,17 @@ def get_assignee(instance, proj_id,work_id): return vals -def comment(instance,proj_id,payload,workid): +def comment(instance, proj_id, payload, workid): + project_val = str(proj_id["project"]) - path ="{}/{}/{}/{}/{}".format("projects",proj_id["project"],"issues",workid,"notes") + if proj_id.get("display_name") == "No Project" or project_val.endswith("/No Project"): + # Epic: use Work Items Notes API (available on all tiers). + # The Epics Notes API (/groups/:id/epics/:iid/notes) is GitLab Premium only. + group_id = project_val.split("/")[0] + path = "{}/{}/{}/{}/{}".format("groups", group_id, "epics", workid, "notes") + else: + # Issue: POST /projects//issues//notes + path = "{}/{}/{}/{}/{}".format("projects", project_val, "issues", workid, "notes") - return instance.post(path,payload) + return instance.post(path, payload)