FAI-399 - Jira stream converters for Faros Destination (#123)

* Start setting up test

* Project converter

* Start sprint converter

* Support multiple records with same id in stream context

* Finish sprint converter

* Fix test

* Add epic records and update issue records

* Remove epicissues

* Start epics converter

* Finish epics converter

* Users converter

* Add config to converters

* Project board ownership

* Issues

* Issue dependencies

* Refactor

* Changelogs

* Update

* TODOs

* Revert "Support multiple records with same id in stream context"

This reverts commit cdc8afc.

* Finally

* Update package.json

* Jira PG

* Move jira resources to own folder

* Move config to StreamContext

* Use existing function

* Use dicts

* Remove extra tests

* Use config object

* Move interfaces to common

* Remove extra converters

* Revert "Remove extra converters"

This reverts commit df8d8eb.

* Add board_issues

* Address comments

destinations/faros-destination/package.json

@@ -32,8 +32,10 @@
   "dependencies": {
     "faros-airbyte-cdk": "^0.1.23",
     "faros-feeds-sdk": "^0.9.0",
+    "git-url-parse": "^11.6.0",
     "jsonata": "^1.8.5",
     "object-sizeof": "^1.6.1",
+    "turndown": "^7.1.1",
     "verror": "^1.10.0"
   },
   "jest": {
@@ -54,5 +56,8 @@
         "tsconfig": "test/tsconfig.json"
       }
     }
+  },
+  "devDependencies": {
+    "@types/turndown": "^5.0.1"
   }
 }

destinations/faros-destination/resources/spec.json

@@ -79,6 +79,26 @@
         "title": "JSONata destination models",
         "description": "Destination models when using JSONata expression.",
         "examples": ["ims_Incident", "vcs_Commit"]
+      },
+      "source_specific_configs": {
+        "type": "object",
+        "title": "Source-specific configs",
+        "description": "Configuration options that apply to specific sources.",
+        "properties": {
+          "jira": {
+            "title": "Jira",
+            "description": "Configuration options that apply to records generated by the Jira Source.",
+            "type": "object",
+            "properties": {
+              "use_board_ownership": {
+                "type": "boolean",
+                "title": "Use Board Ownership",
+                "description": "Use Jira boards for assigning owners of tasks, or use projects.",
+                "default": false
+              }
+            }
+          }
+        }
       }
     }
   }

destinations/faros-destination/src/converters/asana/common.ts

@@ -61,7 +61,10 @@ export class AsanaCommon {
     };
   }
 
-  static tms_TaskBoard(section: AsanaSection, source: string): DestinationRecord {
+  static tms_TaskBoard(
+    section: AsanaSection,
+    source: string
+  ): DestinationRecord {
     return {
       model: 'tms_TaskBoard',
       record: {

destinations/faros-destination/src/converters/asana/projects.ts

@@ -1,5 +1,5 @@
 import {AirbyteRecord} from 'faros-airbyte-cdk';
-import {Utils} from 'faros-feeds-sdk'
+import {Utils} from 'faros-feeds-sdk';
 
 import {DestinationModel, DestinationRecord, StreamContext} from '../converter';
 import {AsanaCommon, AsanaConverter} from './common';
@@ -13,11 +13,11 @@ export class AsanaProjects extends AsanaConverter {
   ): ReadonlyArray<DestinationRecord> {
     const source = this.streamName.source;
     const project = record.record.data;
-    
+
     const tmsProject: DestinationRecord = {
       model: 'tms_Project',
       record: {
-        uid: project.gid, 
+        uid: project.gid,
         name: project.name,
         description: project.notes?.substring(
           0,
@@ -26,8 +26,8 @@ export class AsanaProjects extends AsanaConverter {
         createdAt: Utils.toDate(project.created_at),
         updatedAt: Utils.toDate(project.modified_at),
         source,
-      }
-    }
+      },
+    };
 
     return [tmsProject];
   }

destinations/faros-destination/src/converters/asana/stories.ts

@@ -5,9 +5,7 @@ import {DestinationModel, DestinationRecord, StreamContext} from '../converter';
 import {AsanaCommon, AsanaConverter} from './common';
 
 export class AsanaStories extends AsanaConverter {
-  readonly destinationModels: ReadonlyArray<DestinationModel> = [
-    'tms_Task',
-  ];
+  readonly destinationModels: ReadonlyArray<DestinationModel> = ['tms_Task'];
 
   convert(
     record: AirbyteRecord,

destinations/faros-destination/src/converters/converter.ts

@@ -1,4 +1,4 @@
-import {AirbyteRecord} from 'faros-airbyte-cdk';
+import {AirbyteConfig, AirbyteRecord} from 'faros-airbyte-cdk';
 import {snakeCase} from 'lodash';
 import sizeof from 'object-sizeof';
 import {Dictionary} from 'ts-essentials';
@@ -39,6 +39,8 @@ export abstract class Converter {
 
 /** Stream context to store records by stream */
 export class StreamContext {
+  constructor(readonly config: AirbyteConfig) {}
+
   private readonly recordsByStreamName: Dictionary<Dictionary<AirbyteRecord>> =
     {};
 

destinations/faros-destination/src/converters/jira/application_roles.ts

@@ -0,0 +1,16 @@
+import {AirbyteRecord} from 'faros-airbyte-cdk';
+
+import {DestinationModel, DestinationRecord, StreamContext} from '../converter';
+import {JiraConverter} from './common';
+
+export class JiraApplicationRoles extends JiraConverter {
+  readonly destinationModels: ReadonlyArray<DestinationModel> = []; // TODO: set destination model
+
+  convert(
+    record: AirbyteRecord,
+    ctx: StreamContext
+  ): ReadonlyArray<DestinationRecord> {
+    // TODO: convert records
+    return [];
+  }
+}

destinations/faros-destination/src/converters/jira/avatars.ts

@@ -0,0 +1,16 @@
+import {AirbyteRecord} from 'faros-airbyte-cdk';
+
+import {DestinationModel, DestinationRecord, StreamContext} from '../converter';
+import {JiraConverter} from './common';
+
+export class JiraAvatars extends JiraConverter {
+  readonly destinationModels: ReadonlyArray<DestinationModel> = []; // TODO: set destination model
+
+  convert(
+    record: AirbyteRecord,
+    ctx: StreamContext
+  ): ReadonlyArray<DestinationRecord> {
+    // TODO: convert records
+    return [];
+  }
+}

destinations/faros-destination/src/converters/jira/board_issues.ts

@@ -0,0 +1,28 @@
+import {AirbyteRecord} from 'faros-airbyte-cdk';
+
+import {DestinationModel, DestinationRecord, StreamContext} from '../converter';
+import {JiraConverter} from './common';
+
+export class JiraBoardIssues extends JiraConverter {
+  readonly destinationModels: ReadonlyArray<DestinationModel> = [
+    'tms_TaskBoardRelationship',
+  ];
+
+  convert(
+    record: AirbyteRecord,
+    ctx: StreamContext
+  ): ReadonlyArray<DestinationRecord> {
+    if (!this.useBoardOwnership(ctx)) return [];
+    const issue = record.record.data;
+    const source = this.streamName.source;
+    return [
+      {
+        model: 'tms_TaskBoardRelationship',
+        record: {
+          task: {uid: issue.key, source},
+          board: {uid: String(issue.boardId), source},
+        },
+      },
+    ];
+  }
+}

destinations/faros-destination/src/converters/jira/boards.ts

@@ -0,0 +1,34 @@
+import {AirbyteRecord} from 'faros-airbyte-cdk';
+
+import {DestinationModel, DestinationRecord, StreamContext} from '../converter';
+import {JiraConverter} from './common';
+
+export class JiraBoards extends JiraConverter {
+  readonly destinationModels: ReadonlyArray<DestinationModel> = [
+    'tms_TaskBoard',
+    'tms_TaskBoardProjectRelationship',
+  ];
+
+  convert(
+    record: AirbyteRecord,
+    ctx: StreamContext
+  ): ReadonlyArray<DestinationRecord> {
+    if (!this.useBoardOwnership(ctx)) return [];
+    const board = record.record.data;
+    const uid = board.id.toString();
+    const source = this.streamName.source;
+    return [
+      {
+        model: 'tms_TaskBoard',
+        record: {uid, name: board.name, source},
+      },
+      {
+        model: 'tms_TaskBoardProjectRelationship',
+        record: {
+          board: {uid, source},
+          project: {uid: board.projectKey, source},
+        },
+      },
+    ];
+  }
+}

destinations/faros-destination/src/converters/jira/common.ts

@@ -0,0 +1,74 @@
+import {AirbyteRecord} from 'faros-airbyte-cdk';
+import {Dictionary} from 'ts-essentials';
+
+import {Converter, StreamContext} from '../converter';
+
+export interface Assignee {
+  readonly uid: string;
+  readonly assignedAt: Date;
+}
+
+export enum RepoSource {
+  BITBUCKET = 'Bitbucket',
+  GITHUB = 'GitHub',
+  GITLAB = 'GitLab',
+  VCS = 'VCS',
+}
+
+export interface Repo {
+  readonly source: RepoSource;
+  readonly org: string;
+  readonly name: string;
+}
+
+export interface PullRequest {
+  readonly repo: Repo;
+  readonly number: number;
+}
+
+export interface Status {
+  readonly category: string;
+  readonly detail: string;
+}
+
+export interface SprintIssue {
+  id: number;
+  key: string;
+  fields: Dictionary<any>;
+  issueId: string;
+  sprintId: number;
+}
+
+export class JiraCommon {
+  static POINTS_FIELD_NAMES = ['Story Points', 'Story point estimate'];
+  static DEV_FIELD_NAME = 'Development';
+  static EPIC_LINK_FIELD_NAME = 'Epic Link';
+  static SPRINT_FIELD_NAME = 'Sprint';
+
+  static normalize(str: string): string {
+    return str.replace(/\s/g, '').toLowerCase();
+  }
+}
+
+export const JiraStatusCategories: ReadonlyMap<string, string> = new Map(
+  ['Todo', 'InProgress', 'Done'].map((s) => [JiraCommon.normalize(s), s])
+);
+
+export interface JiraConfig {
+  use_board_ownership?: boolean;
+}
+
+export abstract class JiraConverter extends Converter {
+  /** All Jira records should have id property */
+  id(record: AirbyteRecord): any {
+    return record?.record?.data?.id;
+  }
+
+  protected jiraConfig(ctx: StreamContext): JiraConfig {
+    return ctx.config.source_specific_configs?.jira ?? {};
+  }
+
+  protected useBoardOwnership(ctx: StreamContext): boolean {
+    return this.jiraConfig(ctx).use_board_ownership ?? false;
+  }
+}

destinations/faros-destination/src/converters/jira/dashboards.ts

@@ -0,0 +1,16 @@
+import {AirbyteRecord} from 'faros-airbyte-cdk';
+
+import {DestinationModel, DestinationRecord, StreamContext} from '../converter';
+import {JiraConverter} from './common';
+
+export class JiraDashboards extends JiraConverter {
+  readonly destinationModels: ReadonlyArray<DestinationModel> = []; // TODO: set destination model
+
+  convert(
+    record: AirbyteRecord,
+    ctx: StreamContext
+  ): ReadonlyArray<DestinationRecord> {
+    // TODO: convert records
+    return [];
+  }
+}

destinations/faros-destination/src/converters/jira/epics.ts

@@ -0,0 +1,50 @@
+import {AirbyteRecord} from 'faros-airbyte-cdk';
+import TurndownService from 'turndown';
+
+import {
+  DestinationModel,
+  DestinationRecord,
+  StreamContext,
+  StreamName,
+} from '../converter';
+import {JiraCommon, JiraConverter, JiraStatusCategories} from './common';
+
+export class JiraEpics extends JiraConverter {
+  readonly destinationModels: ReadonlyArray<DestinationModel> = ['tms_Epic'];
+
+  private turndown = new TurndownService();
+
+  convert(
+    record: AirbyteRecord,
+    ctx: StreamContext
+  ): ReadonlyArray<DestinationRecord> {
+    const epic = record.record.data;
+    const source = this.streamName.source;
+    const status = epic.fields.status ?? {};
+    let description = null;
+    if (typeof epic.fields.description === 'string') {
+      description = epic.fields.description;
+    } else if (epic.renderedFields.description) {
+      description = this.turndown.turndown(epic.renderedFields.description);
+    }
+
+    return [
+      {
+        model: 'tms_Epic',
+        record: {
+          uid: epic.key,
+          name: epic.fields.summary ?? null,
+          description,
+          status: {
+            category: JiraStatusCategories.get(
+              JiraCommon.normalize(status.statusCategory?.name)
+            ),
+            detail: status.name,
+          },
+          project: {uid: epic.projectKey, source},
+          source,
+        },
+      },
+    ];
+  }
+}

destinations/faros-destination/src/converters/jira/filter_sharing.ts

@@ -0,0 +1,16 @@
+import {AirbyteRecord} from 'faros-airbyte-cdk';
+
+import {DestinationModel, DestinationRecord, StreamContext} from '../converter';
+import {JiraConverter} from './common';
+
+export class JiraFilterSharing extends JiraConverter {
+  readonly destinationModels: ReadonlyArray<DestinationModel> = []; // TODO: set destination model
+
+  convert(
+    record: AirbyteRecord,
+    ctx: StreamContext
+  ): ReadonlyArray<DestinationRecord> {
+    // TODO: convert records
+    return [];
+  }
+}

destinations/faros-destination/src/converters/jira/filters.ts

@@ -0,0 +1,16 @@
+import {AirbyteRecord} from 'faros-airbyte-cdk';
+
+import {DestinationModel, DestinationRecord, StreamContext} from '../converter';
+import {JiraConverter} from './common';
+
+export class JiraFilters extends JiraConverter {
+  readonly destinationModels: ReadonlyArray<DestinationModel> = []; // TODO: set destination model
+
+  convert(
+    record: AirbyteRecord,
+    ctx: StreamContext
+  ): ReadonlyArray<DestinationRecord> {
+    // TODO: convert records
+    return [];
+  }
+}