diff --git a/issue_migration/README.md b/issue_migration/README.md index bbe8c23..7009eb9 100644 --- a/issue_migration/README.md +++ b/issue_migration/README.md @@ -54,6 +54,8 @@ The script will accept different CLI arguments at the time of calling `main.py`. - `--dry-run` -> Runs the script in dry-mode. Events will not be sent to SaaS but it will print the payload that was generated - `--help` -> Prints help log - It will print out all the CLI arguments that are available +- `--comments` -> Also migrate each issue's historical comments (see [Migrating comments](#migrating-comments)) +- `--sync-grouping` -> Copy the on-prem project's grouping configuration to the SaaS project before replaying issues (see [Syncing grouping configuration](#syncing-grouping-configuration)) There are 2 ways that you can specify the criteria that are used to fetch issues from an on-prem instance: @@ -71,6 +73,41 @@ There are 2 ways that you can specify the criteria that are used to fetch issues **NOTE:** Values specified via the CLI will overwrite values specified in an `.env` file +### Migrating comments +--- +When the `--comments` flag is passed, the script fetches each on-prem issue's comments (`GET /issues/{id}/comments/`) and recreates them on the corresponding SaaS issue (`POST /issues/{id}/comments/`). + +**Important limitation:** The Sentry API does **not** allow setting a comment's author or timestamp. Any comment created through the API is attributed to the owner of `SAAS_AUTH_TOKEN` and stamped with the time it was created. This is enforced server-side (the note endpoint hard-codes the author to the authenticated user and the create serializer accepts no author/timestamp fields), so it cannot be worked around through the public API. + +To avoid losing that information, the script preserves provenance in the comment body. Each migrated comment looks like: + +``` +[Migrated from self-hosted — originally by jane@acme.com on 2024-03-12T14:02:00Z] + + +``` + +Notes: +- Comments are migrated oldest-first so they read chronologically. +- Migration is idempotent: before posting, the script checks existing SaaS comments and skips any whose body already matches, so re-running won't create duplicates. +- If exact author/timestamp fidelity is a hard requirement, it is not achievable via the public API - raise it with the Sentry import/relocation team. +- Combined with `--dry-run`, nothing is written to SaaS; instead the script lists every comment (with its provenance-prefixed body) that would be migrated, reading on-prem only. + +### Syncing grouping configuration +--- +When the `--sync-grouping` flag is passed, the script copies **project-level** grouping configuration from the on-prem project to the SaaS project **before** issues are replayed, so replayed events group the same way they did on-prem. This is a one-time project setting, not per-issue data. + +Applied automatically (via `PUT /projects/{org}/{project}/`): +- `fingerprintingRules` (project option `sentry:fingerprinting_rules`) +- `groupingEnhancements` (project option `sentry:grouping_enhancements`) + +Surfaced in the logs but **not** applied automatically: +- `groupingConfig` / `secondaryGroupingConfig` (the grouping algorithm version). Forcing the SaaS grouping algorithm to an older version can change how events group and is generally discouraged, so review these manually if you need to match them. + +Combined with `--dry-run`, nothing is written to SaaS; instead the script lists which grouping settings would be migrated. + +**Not covered:** UI-side merges are intentionally out of scope. A merge destroys the child groups (the survivor just carries multiple hashes), the script only replays one event per issue, and merges only apply to error-type issues - so they cannot be faithfully reproduced by this script. Track that separately if needed. + ## Things to look out for - If you are migrating over issue assignee information, make sure that the team or person assigned to a ticket in the on-prem instance also exists on SaaS diff --git a/issue_migration/main.py b/issue_migration/main.py index ea349d4..89b2762 100644 --- a/issue_migration/main.py +++ b/issue_migration/main.py @@ -29,12 +29,18 @@ def init(self): self.dry_run = "--dry-run" in cli_args dryable.set(self.dry_run) + self.migrate_comments = "--comments" in cli_args + self.sync_grouping = "--sync-grouping" in cli_args + if self.dry_run: self.logger.debug('Running in dry-mode') - + self.memberObj.populate_members(self.sentry.get_org_members()) self.memberObj.populate_teams(self.sentry.get_org_teams()) + if self.sync_grouping: + self.sync_grouping_config() + filters = utils.get_request_filters(sys.argv, self.logger) if filters is None: raise Exception("Invalid CLI arguments") @@ -48,6 +54,7 @@ def init(self): if metadata is not None: if self.dry_run: self.print_issue_data(metadata) + self.preview_dry_run_comments(metadata) else: self.update_issues(metadata) @@ -83,14 +90,15 @@ def get_issue_metadata(self, issues, metadata): event_id = issue["event_id"] issue_metadata = utils.get_issue_attr(event_id, metadata, "issue_metadata") integration_data = utils.get_issue_attr(event_id, metadata, "integration_data") + onprem_id = utils.get_issue_attr(event_id, metadata, "onprem_id") if issue_metadata is None: self.logger.warn(f'Could not update SaaS issue with ID {issue_id} (Issue created but not updated) - Skipping...') continue - self.update_issue_metadata(issue_id, issue_metadata, integration_data) + self.update_issue_metadata(issue_id, issue_metadata, integration_data, onprem_id) - def update_issue_metadata(self, issue_id, issue_metadata, integration_data): + def update_issue_metadata(self, issue_id, issue_metadata, integration_data, onprem_id = None): response = self.sentry.update_issue(issue_id, issue_metadata) if response is not None and "id" in response: self.logger.info(f'SaaS Issue with ID {issue_id} metadata updated succesfully!') @@ -107,6 +115,8 @@ def update_issue_metadata(self, issue_id, issue_metadata, integration_data): else: self.logger.debug(f'No external issues linked to Issue with ID {issue_id}') + self.migrate_issue_comments(issue_id, onprem_id) + def create_issues_on_sass(self, issues): f = open('./output.json', "w") test_data = [] @@ -213,14 +223,15 @@ def create_issues_on_sass(self, issues): if existingIssueID is not None and not self.dry_run: self.logger.debug(f'Issue already created in SaaS instance with ID {existingIssueID} - Only updating issue with metadata') - self.update_issue_metadata(existingIssueID, issue_metadata, integration_data) + self.update_issue_metadata(existingIssueID, issue_metadata, integration_data, issue["id"]) continue if self.dry_run: obj = { "issue_skeleton" : payload, "issue_metadata" : issue_metadata, - "integration_data" : integration_data + "integration_data" : integration_data, + "onprem_id" : issue["id"] } metadata.append(obj) else: @@ -228,7 +239,8 @@ def create_issues_on_sass(self, issues): obj = { "event_id" : eventResponse["id"], "issue_metadata" : issue_metadata, - "integration_data" : integration_data + "integration_data" : integration_data, + "onprem_id" : issue["id"] } metadata.append(obj) @@ -250,6 +262,140 @@ def create_issues_on_sass(self, issues): def print_issue_data(self, data): self.logger.debug(data, True) + def sync_grouping_config(self): + """Copy project-level grouping config (fingerprint rules + grouping + enhancements) from the on-prem project to the SaaS project. This is a + one-time, project-level operation - it is intentionally NOT part of the + per-issue loop.""" + self.logger.debug('Syncing grouping configuration from on-prem to SaaS') + config = self.sentry.get_on_prem_project_grouping_config() + if config is None: + self.logger.warn('Could not fetch on-prem grouping config - skipping grouping sync') + return + + # Grouping settings that would actually be applied to the SaaS project. + would_apply = [key for key in self.sentry.SAFE_GROUPING_KEYS if config.get(key) is not None] + if self.dry_run: + if len(would_apply) == 0: + self.logger.debug('Dry-run: no fingerprint rules or grouping enhancements found on-prem - nothing would be migrated') + else: + self.logger.info(f'Dry-run: the following grouping settings would be migrated to SaaS: {", ".join(would_apply)}') + + for key in self.sentry.SAFE_GROUPING_KEYS: + value = config.get(key) + preview = (value[:200] + '...') if isinstance(value, str) and len(value) > 200 else value + label = 'would migrate' if (self.dry_run and value is not None) else 'On-prem' + self.logger.debug(f'{label} {key}: {preview!r}') + + # Surface (but don't force) the grouping algorithm version. Overriding + # the SaaS grouping algorithm can change how events group and is + # generally discouraged, so the operator should review this manually. + for key in self.sentry.INFO_GROUPING_KEYS: + if config.get(key): + self.logger.info( + f'On-prem {key} is {config.get(key)!r} - NOT applied automatically. ' + f'Review SaaS project grouping settings if you need to match it.' + ) + + if self.dry_run: + self.logger.debug('Dry-run: grouping config will not be written to SaaS') + return + + result = self.sentry.set_saas_project_grouping_config(config) + if result is not None and result.get("skipped"): + self.logger.warn('No fingerprint rules or grouping enhancements found on-prem - nothing to sync') + elif result is not None: + self.logger.info('SaaS project grouping configuration updated successfully!') + else: + self.logger.error('Could not update SaaS project grouping configuration') + + def preview_dry_run_comments(self, metadata): + """Dry-run preview: list the comments that WOULD be migrated. Only + previews issues that passed normalization (i.e. that would actually be + created on SaaS), mirroring the real run. Reads on-prem comments only + (read-only) - nothing is written to SaaS.""" + if not self.migrate_comments: + return + + self.logger.debug('Dry-run: previewing comments that would be migrated') + total = 0 + for data in metadata: + onprem_id = data.get("onprem_id") + if onprem_id is None: + continue + + comments = self.sentry.get_on_prem_issue_comments(onprem_id) + if comments is None: + self.logger.warn(f'Dry-run: could not fetch comments for on-prem issue {onprem_id}') + continue + if len(comments) == 0: + continue + + comments = sorted(comments, key=lambda c: c.get("dateCreated") or "") + self.logger.debug(f'On-prem issue {onprem_id}: {len(comments)} comment(s) would be migrated') + for comment in comments: + body = utils.build_migrated_comment_body(comment) + if body is None: + continue + total = total + 1 + self.logger.debug(f'--- Comment that would be migrated (on-prem issue {onprem_id}) ---\n{body}') + + if total == 0: + self.logger.debug('Dry-run: no comments found to migrate') + else: + self.logger.info( + f'Dry-run: {total} comment(s) would be migrated in total ' + f'(original author/timestamp preserved in the comment body, not natively)' + ) + + def migrate_issue_comments(self, saas_group_id, onprem_id): + """Replay an on-prem issue's comments onto the newly created SaaS issue. + + IMPORTANT: The Sentry API attributes every created note to the auth + token's owner and stamps it at creation time - the original author and + timestamp cannot be set via the API. We therefore preserve provenance + by prepending it to the comment body.""" + if not self.migrate_comments or onprem_id is None: + return + + source_comments = self.sentry.get_on_prem_issue_comments(onprem_id) + if source_comments is None: + self.logger.warn(f'Could not fetch comments for on-prem issue {onprem_id} - skipping comments') + return + if len(source_comments) == 0: + return + + # Oldest first so migrated notes read in chronological order. + source_comments = sorted(source_comments, key=lambda c: c.get("dateCreated") or "") + + # Skip comments already migrated (idempotent across re-runs). + existing_texts = set() + if not self.dry_run: + for existing in self.sentry.get_saas_issue_comments(saas_group_id): + existing_text = (existing.get("data") or {}).get("text") + if existing_text: + existing_texts.add(existing_text) + + migrated = 0 + for comment in source_comments: + text = utils.build_migrated_comment_body(comment) + if text is None: + continue + if text in existing_texts: + continue + response = self.sentry.create_saas_comment(saas_group_id, text) + if self.dry_run or response is not None: + migrated = migrated + 1 + existing_texts.add(text) + else: + self.logger.error(f'Could not migrate a comment to SaaS issue {saas_group_id}') + + if migrated > 0: + self.logger.info( + f'Migrated {migrated} comment(s) to SaaS issue {saas_group_id} ' + f'(original author/timestamp preserved in the comment body, not natively)' + ) + if __name__ == "__main__": main = Main() main.init() \ No newline at end of file diff --git a/issue_migration/request.py b/issue_migration/request.py index f418477..7919002 100644 --- a/issue_migration/request.py +++ b/issue_migration/request.py @@ -2,20 +2,27 @@ import os from dotenv import load_dotenv -def request(url, method, payload = None): +def request(url, method, payload = None, authenticated = None): load_dotenv() try: token = os.environ["SAAS_AUTH_TOKEN"] if "sentry.io" in url else os.environ["ON_PREM_AUTH_TOKEN"] headers = { "Content-Type": "application/json", } - if method == "GET": + # By default GET/PUT requests are authenticated with a bearer token. + # POST is unauthenticated by default because the store/ingest endpoint + # authenticates via the sentry_key in the URL, not a bearer token. + # Callers that POST to a REST endpoint (e.g. creating a comment) must + # pass authenticated=True. + if authenticated is None: + authenticated = method in ("GET", "PUT") + if authenticated: headers["Authorization"] = "Bearer " + token + if method == "GET": return requests.get(url, headers = headers) elif method == "POST": - return requests.post(url, json = payload) + return requests.post(url, json = payload, headers = headers) elif method == "PUT": - headers["Authorization"] = "Bearer " + token return requests.put(url, json = payload, headers = headers) except Exception as e: raise Exception(f'Could not make request to {url} - Reason: {str(e)}') diff --git a/issue_migration/sentry/Sentry.py b/issue_migration/sentry/Sentry.py index c8c59aa..8a961fe 100644 --- a/issue_migration/sentry/Sentry.py +++ b/issue_migration/sentry/Sentry.py @@ -310,4 +310,113 @@ def update_external_issues(self, issue_id, integration_data, integration_id): def build_discover_query(self, migration_id): url = f'https://{self.saas_options["org_name"]}.sentry.io/issues/?query=+migration_id%3A{migration_id}&referrer=issue-list&statsPeriod=90d' - return url \ No newline at end of file + return url + + # ------------------------------------------------------------------------- + # Grouping / fingerprint configuration (project-level) + # ------------------------------------------------------------------------- + # These are project options, not per-issue data. They are read from the + # on-prem project and written to the SaaS project once, before issues are + # replayed, so replayed events group the same way they did on-prem. + + # Config keys that map cleanly to project-update body params and can be + # written safely to SaaS. + SAFE_GROUPING_KEYS = ["fingerprintingRules", "groupingEnhancements"] + # Additional keys we surface for the operator but do NOT write automatically, + # because forcing the SaaS grouping algorithm version can change how events + # group and is generally discouraged. + INFO_GROUPING_KEYS = ["groupingConfig", "secondaryGroupingConfig"] + + def get_on_prem_project_grouping_config(self): + url = f'{self.on_prem_options["url"]}projects/{self.on_prem_options["org_name"]}/{self.on_prem_options["project_name"]}/' + response = request(url, method = "GET") + if response is not None and response.status_code == 200: + data = response.json() + config = {} + for key in self.SAFE_GROUPING_KEYS + self.INFO_GROUPING_KEYS: + config[key] = data.get(key) + return config + + if response is not None: + print(response.json()) + return None + + def set_saas_project_grouping_config(self, config): + payload = {} + for key in self.SAFE_GROUPING_KEYS: + value = config.get(key) + # Send empty strings too (an operator may want to clear a rule set), + # but skip keys that were absent from the on-prem response. + if value is not None: + payload[key] = value + + if len(payload) == 0: + return {"skipped": True} + + url = f'{self.saas_options["url"]}projects/{self.saas_options["org_name"]}/{self.saas_options["project_name"]}/' + response = request(url, method = "PUT", payload = payload) + if response is not None and response.status_code == 200: + return response.json() + + if response is not None: + print(response.json()) + return None + + # ------------------------------------------------------------------------- + # Issue comments (notes) + # ------------------------------------------------------------------------- + # NOTE: The public Sentry API does not allow setting a comment's author or + # timestamp - a created note is always attributed to the auth-token owner + # and stamped at creation time. To preserve provenance we prepend the + # original author + timestamp to the comment body (see main.py). + + def get_on_prem_issue_comments(self, issue_id): + url = f'{self.on_prem_options["url"]}issues/{issue_id}/comments/' + comments = [] + response = request(url, method = "GET") + if response is None or response.status_code != 200: + if response is not None: + print(response.json()) + return None + + comments = comments + response.json() + next = response.links.get('next', {}).get('results') == 'true' + while next: + url = response.links.get('next', {}).get('url') + response = request(url, method = "GET") + if response is None or response.status_code != 200: + break + comments = comments + response.json() + next = response.links.get('next', {}).get('results') == 'true' + + return comments + + def get_saas_issue_comments(self, group_id): + url = f'{self.saas_options["url"]}issues/{group_id}/comments/' + comments = [] + response = request(url, method = "GET") + if response is None or response.status_code != 200: + return comments + + comments = comments + response.json() + next = response.links.get('next', {}).get('results') == 'true' + while next: + url = response.links.get('next', {}).get('url') + response = request(url, method = "GET") + if response is None or response.status_code != 200: + break + comments = comments + response.json() + next = response.links.get('next', {}).get('results') == 'true' + + return comments + + @dryable.Dryable() + def create_saas_comment(self, group_id, text): + url = f'{self.saas_options["url"]}issues/{group_id}/comments/' + response = request(url, method = "POST", payload = {"text": text}, authenticated = True) + if response is not None and response.status_code in [200, 201]: + return response.json() + + if response is not None: + print(response.json()) + return None \ No newline at end of file diff --git a/issue_migration/sentry/utils.py b/issue_migration/sentry/utils.py index 64d2f03..1e0de17 100644 --- a/issue_migration/sentry/utils.py +++ b/issue_migration/sentry/utils.py @@ -81,7 +81,7 @@ def get_dry_run(args): return False def process_cli_args(args, logger): - valid_args = ["--dry-run", "--start", "--end", "--issues", "--fetchRelease"] + valid_args = ["--dry-run", "--start", "--end", "--issues", "--fetchRelease", "--comments", "--sync-grouping"] if "--help" in args: print_help_log() return False @@ -106,6 +106,12 @@ def print_help_log(): }, { "--issues" : "\tList of issues to migrate from on-prem to SaaS" + }, + { + "--comments" : "\tMigrate each issue's historical comments as provenance-prefixed notes (author/timestamp preserved in the comment body, not natively)" + }, + { + "--sync-grouping" : "Copy fingerprint rules and grouping enhancements from the on-prem project to the SaaS project before replaying issues" } ] print('ARGUMENT \t DESCRIPTION') @@ -116,6 +122,21 @@ def print_help_log(): print(f'{key}{i[key]}') +def build_migrated_comment_body(comment): + """Build a SaaS comment body that preserves the original author and + timestamp in text, since the API cannot set them natively.""" + if comment is None: + return None + text = (comment.get("data") or {}).get("text") or comment.get("text") + if not text: + return None + author = "unknown user" + user = comment.get("user") + if isinstance(user, dict): + author = user.get("email") or user.get("name") or user.get("username") or author + timestamp = comment.get("dateCreated") or "unknown time" + return f'[Migrated from self-hosted — originally by {author} on {timestamp}]\n\n{text}' + def replace_all(str, chars, new_val = ""): for char in chars: str = str.replace(char, new_val)