diff --git a/.gitignore b/.gitignore index 8eac626..cf7b8aa 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,5 @@ venv .*.sw? -.env \ No newline at end of file +.env +.DS_Store diff --git a/app/data_sources/azuredevops.py b/app/data_sources/azuredevops.py new file mode 100644 index 0000000..372f618 --- /dev/null +++ b/app/data_sources/azuredevops.py @@ -0,0 +1,123 @@ +import datetime +import logging +from typing import Dict, List +import requests +import base64 +import urllib.parse +from dataclasses import dataclass + +from azure.devops.connection import Connection +from msrest.authentication import BasicAuthentication + +from data_source_api.base_data_source import BaseDataSource, ConfigField, HTMLInputType +from data_source_api.basic_document import DocumentType, BasicDocument +from data_source_api.exception import InvalidDataSourceConfig +from index_queue import IndexQueue +from parsers.html import html_to_text +from data_source_api.utils import parse_with_workers + +logger = logging.getLogger(__name__) + +@dataclass +class DevOpsConfig(): + organization_url: str + access_token: str + project_name: str + query_id: str + + def __post_init__(self): + self.query_id = self.query_id.strip() + self.access_token = self.access_token.strip() + self.project_name = self.project_name.strip() + self.organization_url = self.organization_url.strip() + +class AzuredevopsDataSource(BaseDataSource): + @staticmethod + def get_config_fields() -> List[ConfigField]: + return [ + ConfigField(label="AzureDevOps organization URL", placeholder="https://dev.azure.com/org", name="organization_url"), + ConfigField(label="Personal Access Token", name="access_token", type=HTMLInputType.PASSWORD), + ConfigField(label="Project Name", name="project_name"), + ConfigField(label="Query ID", name="query_id"), + ] + + @classmethod + def get_display_name(cls) -> str: + return "Azure DevOps" + + @staticmethod + def validate_config(config: Dict) -> None: + try: + devops_config = DevOpsConfig(**config) + credentials = BasicAuthentication('', devops_config.access_token) + connection = Connection(base_url=devops_config.organization_url, creds=credentials) + core_client = connection.clients.get_core_client() + core_client.get_projects() + except Exception as e: + raise InvalidDataSourceConfig from e + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._devops_config = DevOpsConfig(**self._config) + credentials = BasicAuthentication('', self._devops_config.access_token) + connection = Connection(base_url=self._devops_config.organization_url, creds=credentials) + self._work_item_tracking_client = connection.clients.get_work_item_tracking_client() + + def _parse_documents_worker(self, raw_docs: List[Dict]): + logging.info(f'Worker parsing {len(raw_docs)} documents') + parsed_docs = [] + total_fed = 0 + for item in raw_docs: + for raw_page in item['comments']: + create_date = datetime.datetime.strptime(raw_page['createdDate'], "%Y-%m-%dT%H:%M:%S.%fZ") + if create_date < self._last_index_time: + continue + author = raw_page['createdBy']['displayName'] + workitem_id = raw_page['workItemId'] + title = str(raw_page['workItemId']) + ' - ' + raw_page['createdBy']['displayName'] + html_content = raw_page['text'] + plain_text = html_to_text(html_content) + author_image_url = raw_page['createdBy']['_links']['avatar']['href'] + url = f"{self._devops_config.organization_url}/{urllib.parse.quote(self._devops_config.project_name)}/_workitems/edit/{raw_page['workItemId']}".strip() + + parsed_docs.append(BasicDocument( + id=workitem_id, + data_source_id=self._data_source_id, + author=author, + author_image_url=author_image_url, + content=plain_text, + type=DocumentType.COMMENT, + title=title, + timestamp=create_date, + location=self._devops_config.project_name, + url=url + )) + + if len(parsed_docs) >= 50: + total_fed += len(parsed_docs) + IndexQueue.get_instance().put(docs=parsed_docs) + parsed_docs = [] + + IndexQueue.get_instance().put(docs=parsed_docs) + total_fed += len(parsed_docs) + if total_fed > 0: + logging.info(f'Worker fed {total_fed} documents') + + + def _list_work_item_comments(self, work_item_url) -> List[Dict]: + authorization = str(base64.b64encode(bytes(':'+self._devops_config.access_token, 'ascii')), 'ascii') + headers = { + 'Accept': 'application/json', + 'Authorization': 'Basic '+authorization + } + return requests.get(url=work_item_url + '/comments', headers=headers).json() + + def _feed_new_documents(self) -> None: + logger.info('Feeding new Azure DevOps Work Items') + raw_docs = [] + work_item_results = self._work_item_tracking_client.query_by_id(self._devops_config.query_id) + for work_item in work_item_results.work_items: + result = self._list_work_item_comments(work_item.url) + if result['totalCount'] > 0: + raw_docs.append(result) + parse_with_workers(self._parse_documents_worker, raw_docs) \ No newline at end of file diff --git a/app/data_sources/google_drive.py b/app/data_sources/google_drive.py index a54c6ec..92f60a5 100644 --- a/app/data_sources/google_drive.py +++ b/app/data_sources/google_drive.py @@ -181,7 +181,7 @@ def _index_files_from_drive(self, drive) -> List[dict]: title=file['name'], content=content, author=file['lastModifyingUser']['displayName'], - author_image_url=file['lastModifyingUser']['photoLink'], + author_image_url=file['lastModifyingUser'].get('photoLink'), location=parent_name, url=file['webViewLink'], timestamp=last_modified, diff --git a/app/requirements.txt b/app/requirements.txt index 08c5b93..bb40396 100644 --- a/app/requirements.txt +++ b/app/requirements.txt @@ -22,3 +22,5 @@ python-pptx alembic rocketchat-API mattermostdriver +persistqueue +azure-devops diff --git a/app/static/data_source_icons/azuredevops.png b/app/static/data_source_icons/azuredevops.png new file mode 100644 index 0000000..504f338 Binary files /dev/null and b/app/static/data_source_icons/azuredevops.png differ diff --git a/docs/data-sources/azuredevops/AddressBar.png b/docs/data-sources/azuredevops/AddressBar.png new file mode 100644 index 0000000..cd39b59 Binary files /dev/null and b/docs/data-sources/azuredevops/AddressBar.png differ diff --git a/docs/data-sources/azuredevops/Gerev.png b/docs/data-sources/azuredevops/Gerev.png new file mode 100644 index 0000000..ff1bcf4 Binary files /dev/null and b/docs/data-sources/azuredevops/Gerev.png differ diff --git a/docs/data-sources/azuredevops/NewButton.png b/docs/data-sources/azuredevops/NewButton.png new file mode 100644 index 0000000..740da0b Binary files /dev/null and b/docs/data-sources/azuredevops/NewButton.png differ diff --git a/docs/data-sources/azuredevops/QuerySelect.png b/docs/data-sources/azuredevops/QuerySelect.png new file mode 100644 index 0000000..eb97c7b Binary files /dev/null and b/docs/data-sources/azuredevops/QuerySelect.png differ diff --git a/docs/data-sources/azuredevops/Settings.png b/docs/data-sources/azuredevops/Settings.png new file mode 100644 index 0000000..0746aa6 Binary files /dev/null and b/docs/data-sources/azuredevops/Settings.png differ diff --git a/docs/data-sources/azuredevops/TokenForm.png b/docs/data-sources/azuredevops/TokenForm.png new file mode 100644 index 0000000..49308a3 Binary files /dev/null and b/docs/data-sources/azuredevops/TokenForm.png differ diff --git a/docs/data-sources/azuredevops/azuredevops.md b/docs/data-sources/azuredevops/azuredevops.md new file mode 100644 index 0000000..134c1a6 --- /dev/null +++ b/docs/data-sources/azuredevops/azuredevops.md @@ -0,0 +1,16 @@ +# Setting up Azure DevOps Data Source +Please note this will only index Work Items returned by your Query + +1. Navigate to the settings menu in the top-right hand corner of the screen and select Personal Access Tokens. +![Settings](./Settings.png) +2. Click on the new option. +![NewButton](./NewButton.png) +1. Complete the form, and set the expiration date to custom. Select the furthest expiration date possible. Make sure to only provide read only permissions. +![TokenForm](./TokenForm.png) +1. Hit Create and copy the token from the next window. +2. Navigate to your Project and go to Boards > Queries. Select your query from the list. +![QuerySelect](./QuerySelect.png) +1. Copy the Query ID from the URL in the address bar. +![AddressBar](./AddressBar.png) +1. Go to Gerev and input all of the data into the fields. +![Gerev](./Gerev.png) \ No newline at end of file diff --git a/ui/src/components/data-source-panel.tsx b/ui/src/components/data-source-panel.tsx index 7e2c1d3..f18bda7 100644 --- a/ui/src/components/data-source-panel.tsx +++ b/ui/src/components/data-source-panel.tsx @@ -297,6 +297,21 @@ export default class DataSourcePanel extends React.ComponentNote that the url must begin with either http:// or https://

)} + + { + this.state.selectedDataSource.value === 'azuredevops' && ( + + 1. {'Go to your Azure DevOps -> top-right Person with the Gear -> Personal Access Token'} + 2. {'Fill out the details, and set your expiration date'} + 3. {"Make sure Read under Work Items is checked and save your token"} + 4. {"Copy the token and paste it here"} + 5. {"Add your Project Name here"} + 6. {"Navigate to Your Project then Boards > Queries"} + 7. {"Select the query that you want from the list and get the id from the end of the URL"} + ex. {"https://dev.azure.com/{org}/{project}/_queries/query/{query_id}/"} + + ) + }