diff --git a/.gitignore b/.gitignore index d5d7504..25a49e5 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ __pycache__/ /build /dist /*.egg-info + +/wheels diff --git a/README.md b/README.md index d89a797..ddbebcb 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,13 @@ rdmo-plugins-github =========== -This repo implements two plugins for [RDMO](https://github.com/rdmorganiser/rdmo): +This repo implements three plugins for [RDMO](https://github.com/rdmorganiser/rdmo): * an [issue provider](https://rdmo.readthedocs.io/en/latest/plugins/index.html#issue-providers), which lets users push their tasks from RDMO to GitHub issues. -* a [project import plugins](https://rdmo.readthedocs.io/en/latest/plugins/index.html#project-import-plugins), which can be used to import projects from (public or private)repos. +* an [import provider](https://rdmo.readthedocs.io/en/latest/plugins/index.html#project-import-plugins), which can be used to import projects from (public or private) repositories as well as repository metadata (dependecy graph, languages, license or CITATION). +* an [export provider](https://rdmo.readthedocs.io/en/latest/plugins/index.html#project-export-plugins), which can be used to export projects to (public or private) repositories. For SMP projects, this plugin also provides other export choices that reuse project data (e.g. README, CITATION or LICENSE files). -The plugin uses [OAUTH 2.0](https://oauth.net/2/), so that users use their respective accounts in both systems. +The plugins use [OAUTH 2.0](https://oauth.net/2/), so that users use their respective accounts in both systems. Setup @@ -15,17 +16,19 @@ Setup Install the plugin in your RDMO virtual environment using pip (directly from GitHub): ```bash -pip install git+https://github.com/rdmorganiser/rdmo-plugins-github +pip install git+https://github.com/MPDL/rdmo-plugins-github@dev ``` -An *App* has to be registered with GitHub. Go to https://github.com/settings/developers and create an application with your RDMO URL as callback URL. +An *App* has to be registered with GitHub. Go to https://github.com/settings/developers and create an application with your RDMO URL as callback URL. GitHub offers two types of apps: [GitHub Apps](https://docs.github.com/en/apps/using-github-apps/about-using-github-apps) and [OAuth Apps](https://docs.github.com/en/apps/oauth-apps), both app types use [OAUTH 2.0](https://oauth.net/2/). -The `client_id` and the `client_secret` need to be configured in `config/settings/local.py`: +The `client_id`, the `client_secret`, the `app_type` (`oauth_app` or `github_app`), and the `github_app_name` ([Register a GitHub App](https://docs.github.com/en/apps/creating-github-apps/registering-a-github-app/registering-a-github-app)) need to be configured in `config/settings/local.py`: ```python GITHUB_PROVIDER = { 'client_id': '', - 'client_secret': '' + 'client_secret': '', + 'app_type': 'oauth_app' | 'github_app', + 'github_app_name': '' } ``` @@ -33,28 +36,48 @@ For the issue provider, add the plugin to `PROJECT_ISSUE_PROVIDERS` in `config/s ```python PROJECT_ISSUE_PROVIDERS += [ - ('github', _('GitHub Provider'), 'rdmo_github.providers.GitHubProvider'), + ('github', _('GitHub Provider'), 'rdmo_github.providers.exports.GitHubIssueProvider'), ] ``` -For the import, add the plugin to `PROJECT_IMPORTS` in `config/settings/local.py`: +For the import, add the plugin to `PROJECT_IMPORTS` and its key to `PROJECT_IMPORTS_LIST` in `config/settings/local.py`: ```python PROJECT_IMPORTS = [ - ('github', _('Import from GitHub'), 'rdmo_github.providers.GitHubImport'), + ('github', _('GitHub'), 'rdmo_github.providers.imports.GitHubImport'), +] + +PROJECT_IMPORTS_LIST += ['github'] +``` + +For the export, add the plugin to `PROJECT_EXPORTS` in `config/settings/local.py`: + +```python +PROJECT_EXPORTS += [ + ('github', _('Github'), 'rdmo_github.providers.exports.GitHubExportProvider'), ] ``` +The export and import plugins use the plugin [rdmo_maus](https://github.com/MPDL/rdmo-plugins-maus). This plugin provides the SMP specific export choices as well as a custom field used in their form templates. Install rdmo_maus in your RDMO virtual environment using pip (directly from GitHub): + +```bash +pip install git+https://github.com/MPDL/rdmo-plugins-maus +``` + Usage ----- ### Issue provider -Users can add a GitHub intergration to their projects. They need to provide the URL to their repository. Afterwards, issues can be pushed to the GitHub repo. +Users can add a GitHub intergration to their projects. They need to provide the URL to their repository. Afterward, project tasks can be pushed to the GitHub repository as issues. Additionally, a secret can be added to enable GitHub to communicate to RDMO when an issue has been closed. For this, a webhook has to be added at `//settings/hooks`. The webhook has to point to `https:///projects//integrations//webhook/`, the content type is `application/json` and the secret has to be exactly the secret entered in the integration. ### Project import -Users can import project import files directly from a public or private GitHub repository. +Users can import xml project files as well as repository metadata (dependency graph, languages, license or CITATION file) directly from a public or private GitHub repository. The metadata import is optimized for SMP projects; i.e. other catalogs may not have matching questions (and attributes) for the metadata or use different attributes. In those cases, the plugin imports only a subset of the metadata found in the repository. + +### Project export + +Users can export project files directly to a public or private GitHub repository. For SMP projects, they can also export custom files (README, CITATION, LICENSE) created with the SMP project's data. They can choose to export to an existing repository or to create a new one. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 510d921..ba6863e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,13 +13,12 @@ maintainers = [ description = "GitHub Plugins for RDMO." readme = "README.md" requires-python = ">=3.8" -license = {file = "LICENSE"} +license-files = ["LICENSE"] classifiers = [ 'Development Status :: 5 - Production/Stable', 'Environment :: Web Environment', 'Framework :: Django :: 2.2', 'Intended Audience :: Science/Research', - 'License :: OSI Approved :: Apache Software License', 'Operating System :: OS Independent', "Programming Language :: Python :: 3 :: Only", 'Programming Language :: Python :: 3.8', @@ -36,8 +35,11 @@ dynamic = ["version"] [project.urls] repository = "https://github.com/rdmorganiser/rdmo-plugins-github" -[tool.setuptools] -packages = ["rdmo_github"] +[tool.setuptools.packages.find] +include = ["rdmo_github*"] + +[tool.setuptools.package-data] +"*" = ["*"] [tool.setuptools.dynamic] version = {attr = "rdmo_github.__version__"} diff --git a/rdmo_github/forms/__init__.py b/rdmo_github/forms/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/rdmo_github/forms/forms.py b/rdmo_github/forms/forms.py new file mode 100644 index 0000000..4e4e096 --- /dev/null +++ b/rdmo_github/forms/forms.py @@ -0,0 +1,188 @@ +from django import forms +from django.core.exceptions import ValidationError +from django.templatetags.static import static +from django.utils.html import format_html +from django.utils.translation import gettext_lazy as _ + +from rdmo_maus.forms.fields import MultivalueCheckboxMultipleChoiceField + +from .validators import validate_import_file_path, validate_new_repo_name + + +class GithubBaseForm(forms.Form): + def __init__(self, *args, **kwargs): + repo_choices = kwargs.pop('repo_choices') + repo_help_text = kwargs.pop('repo_help_text') + super().__init__(*args, **kwargs) + + if repo_choices is not None: + self.fields['repo'].choices = repo_choices + + if repo_help_text is not None: + self.fields['repo'].help_text = repo_help_text + +class GitHubExportForm(GithubBaseForm): + def __init__(self, *args, **kwargs): + repo_choices = kwargs.get('repo_choices') + export_choices = kwargs.pop('export_choices', None) + super().__init__(*args, **kwargs) + + if repo_choices is not None and len(repo_choices) == 0: + self.fields['new_repo'].initial = True + + if export_choices is not None: + self.fields['exports'].choices = export_choices.get('choices') + self.fields['branch'].widget = forms.TextInput( + attrs={'oninput': f"hideAllChoiceWarningMessages(this, {len(export_choices.get('choices'))})"} + ) + self.fields['exports'].choice_validators = export_choices.get('choice_validators', {}) + self.fields['exports'].widget.choice_attributes = export_choices.get('choice_attributes', {}) + self.fields['exports'].widget.choice_warnings = export_choices.get('choice_warnings', {}) + + new_repo = forms.BooleanField( + label=_('Create new repository'), + required=False, + widget=forms.CheckboxInput( + attrs={ + 'onclick': 'toggleRepoFields("id_new_repo", "form-group field-new_repo_name", "form-group field-repo")' + }) + ) + + new_repo_name = forms.CharField( + label=_('Name for the new repository'), + help_text=_( + 'This name must be unique, otherwise the export will fail.' + ), + required=False, + widget=forms.TextInput(attrs={'placeholder': _('example-repo-name')}), + validators=[validate_new_repo_name] + ) + + repo = forms.ChoiceField( + label=_('GitHub repository'), + required=False, + widget=forms.RadioSelect + ) + + exports = MultivalueCheckboxMultipleChoiceField( + label=_('Export choices'), + help_text=_('Warning: Existing content in GitHub will be overwritten.'), + include_select_all_choice=True + ) + + branch = forms.CharField( + label=_('Branch'), + help_text=_('An existing branch in the GitHub repository. For a new repository it must be "main".'), + initial='main' + ) + + commit_message = forms.CharField(label=_('Commit message')) + + class Media: + script_tag = ''.format( + cbId='id_new_repo', + cC='form-group field-new_repo_name', + uC='form-group field-repo' + ) + js = [format_html(script_tag, static('plugins/js/github_form.js'))] + + def clean(self): + super().clean() + new_repo = self.cleaned_data.get('new_repo') + new_repo_name = self.cleaned_data.get('new_repo_name') + repo = self.cleaned_data.get('repo') + + if new_repo and new_repo_name == '': + self.add_error( + 'new_repo_name', + ValidationError(_('A name for the new repository is required.'), code='required') + ) + + if not new_repo and 'new_repo_name' in self.errors: # ignore new_repo_errors because repo will be used instead + self._errors.pop('new_repo_name') + + if not new_repo and repo == '': + self.add_error('repo', ValidationError(_('A GitHub repository is required.'), code='required')) + +class GitHubImportForm(GithubBaseForm): + def __init__(self, *args, **kwargs): + repo_choices = kwargs.get('repo_choices') + import_choices = kwargs.pop('import_choices', None) + super().__init__(*args, **kwargs) + + if repo_choices is not None and len(repo_choices) == 0: + self.fields['other_repo_check'].initial = True + + if import_choices is not None: + self.fields['imports'].choices = import_choices.get('choices') + self.fields['imports'].choice_validators = import_choices.get('choice_validators', {}) + self.fields['imports'].widget.choice_attributes = import_choices.get('choice_attributes', {}) + self.fields['imports'].widget.choice_warnings = import_choices.get('choice_warnings', {}) + else: + self.fields['imports'] = forms.CharField( + label=_('File path'), + help_text=_("The import file's relative path in the repository. The file must be in XML format."), + widget=forms.TextInput(attrs={'placeholder': _('example_folder/example_xml_file.xml')}), + validators=[validate_import_file_path] + ) + + other_repo_check = forms.BooleanField( + label=_('Use other repository'), + required=False, + widget=forms.CheckboxInput( + attrs={ + 'onclick': 'toggleRepoFields("{cbId}", "{cC}", "{uC}")'.format( + cbId='id_other_repo_check', + cC='form-group field-other_repo', + uC='form-group field-repo' + ) + } + ) + ) + + repo = forms.ChoiceField( + label=_('GitHub repository'), + required=False, + widget=forms.RadioSelect + ) + + other_repo = forms.CharField( + label=_('GitHub repository'), + help_text=_( + 'URL of the repository you want to import from. It must be either public, or accesible to you.' + ), + widget=forms.TextInput(attrs={'placeholder': _('https://github.com/example-owner/example-repo')}), + required=False + ) + + imports = MultivalueCheckboxMultipleChoiceField( + label=_('Import choices'), + help_text=_('Select the import choices. Once they are in the gray box, move them to prioritize them.'), + sortable=True + ) + + ref = forms.CharField( + label=_('Branch, tag, or commit'), + initial='main' + ) + + class Media: + script_tag = ''.format( + cbId='id_other_repo_check', + cC='form-group field-other_repo', + uC='form-group field-repo' + ) + js = [format_html(script_tag, static('plugins/js/github_form.js'))] + + def clean(self): + super().clean() + other_repo_check = self.cleaned_data.get('other_repo_check') + other_repo = self.cleaned_data.get('other_repo') + repo = self.cleaned_data.get('repo') + + if other_repo_check and other_repo == '': + self.add_error('other_repo', ValidationError(_('A GitHub repository is required.'), code='required')) + + if not other_repo_check and repo == '': + self.add_error('repo', ValidationError(_('A GitHub repository is required.'), code='required')) + diff --git a/rdmo_github/forms/validators.py b/rdmo_github/forms/validators.py new file mode 100644 index 0000000..6f9bdd1 --- /dev/null +++ b/rdmo_github/forms/validators.py @@ -0,0 +1,21 @@ +from django.core.exceptions import ValidationError +from django.utils.translation import gettext_lazy as _ + +from rdmo_maus.forms.validators import validate_text_field + + +def validate_new_repo_name(value): + field_name = _('Repository name') + min_length = 1 + max_length = 50 + not_allowed_pattern = r'[^A-Za-z0-9\-\_\.]' + allowed_char_name_str = _('alphanumeric, hyphen, underscore, and period') + + return validate_text_field(field_name, value, min_length, max_length, not_allowed_pattern, allowed_char_name_str) + +def validate_import_file_path(value): + if not value.endswith('.xml'): + raise ValidationError( + _('File must be in XML format.'), + code='invalid' + ) diff --git a/rdmo_github/locale/de/LC_MESSAGES/django.mo b/rdmo_github/locale/de/LC_MESSAGES/django.mo new file mode 100644 index 0000000..9472fb7 Binary files /dev/null and b/rdmo_github/locale/de/LC_MESSAGES/django.mo differ diff --git a/rdmo_github/locale/de/LC_MESSAGES/django.po b/rdmo_github/locale/de/LC_MESSAGES/django.po new file mode 100644 index 0000000..e8aa0da --- /dev/null +++ b/rdmo_github/locale/de/LC_MESSAGES/django.po @@ -0,0 +1,360 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-11 11:48+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" + +#: forms/forms.py:43 +msgid "Create new repository" +msgstr "Neues Repositorium anlegen" + +#: forms/forms.py:52 +msgid "Name for the new repository" +msgstr "Name des neuen Repositoriums" + +#: forms/forms.py:54 +msgid "This name must be unique, otherwise the export will fail." +msgstr "Dieser Name muss eindeutig sein, andernfalls schlägt der Export fehl." + +#: forms/forms.py:57 +msgid "example-repo-name" +msgstr "beispiel-repo-name" + +#: forms/forms.py:62 forms/forms.py:144 forms/forms.py:150 +msgid "GitHub repository" +msgstr "GitHub-Repositorium" + +#: forms/forms.py:68 +msgid "Export choices" +msgstr "Optionen zum Exportieren" + +#: forms/forms.py:69 +msgid "Warning: Existing content in GitHub will be overwritten." +msgstr "Warnung: Bestehende Inhalte in GitHub werden überschrieben." + +#: forms/forms.py:74 +msgid "Branch" +msgstr "Branch" + +#: forms/forms.py:75 +msgid "" +"An existing branch in the GitHub repository. For a new repository it must be " +"\"main\"." +msgstr "" +"Ein bestehender Branch im GitHub-Repositorium. Bei einem neuen Repositorium " +"muss er „main“ sein." + +#: forms/forms.py:79 +msgid "Commit message" +msgstr "Commitnachricht" + +#: forms/forms.py:98 +msgid "A name for the new repository is required." +msgstr "Ein Name für das neue Repositorium ist erforderlich." + +#: forms/forms.py:105 forms/forms.py:184 forms/forms.py:187 +msgid "A GitHub repository is required." +msgstr "Ein GitHub-Repositorium ist erforderlich." + +#: forms/forms.py:123 providers/exports.py:41 providers/exports.py:42 +#: providers/exports.py:45 providers/exports.py:48 +msgid "File path" +msgstr "Dateipfad" + +#: forms/forms.py:124 +msgid "" +"The import file's relative path in the repository. The file must be in XML " +"format." +msgstr "" +"Der relative Pfad der Importdatei im Repositorium. Die Datei muss im XML-" +"Format vorliegen." + +#: forms/forms.py:125 providers/exports.py:67 +msgid "example_folder/example_xml_file.xml" +msgstr "beispiel_ordner/beispiel_xml_datei.xml" + +#: forms/forms.py:130 +msgid "Use other repository" +msgstr "Ein anderes Repositorium benutzen" + +#: forms/forms.py:152 +msgid "" +"URL of the repository you want to import from. It must be either public, or " +"accesible to you." +msgstr "" +"URL des GitHub-Repositoriums, aus dem Sie importieren möchten. Es muss " +"entweder öffentlich oder für Sie zugänglich sein." + +#: forms/forms.py:154 +msgid "https://github.com/example-owner/example-repo" +msgstr "https://github.com/beispiel-besitzerin/beispiel-repo" + +#: forms/forms.py:159 +msgid "Import choices" +msgstr "Optionen zum Importieren" + +#: forms/forms.py:160 +msgid "" +"Select the import choices. Once they are in the gray box, move them to " +"prioritize them." +msgstr "" +"Wählen Sie die Optionen zum Importieren aus. Sobald sie sich im " +"grauen Bereich befinden, verschieben Sie sie, um sie zu priorisieren." + +#: forms/forms.py:165 +msgid "Branch, tag, or commit" +msgstr "Branch, Tag, oder Commit" + +#: forms/validators.py:8 +msgid "Repository name" +msgstr "Name des Repositoriums" + +#: forms/validators.py:12 +msgid "alphanumeric, hyphen, underscore, and period" +msgstr "alphanumerisch, Bindestrich, Unterstrich und Punkt" + +#: forms/validators.py:19 +msgid "File must be in XML format." +msgstr "Die Datei muss im XML-Format vorliegen." + +#: mixins.py:224 +msgid "Request (export or import) error" +msgstr "Anfragefehler (Export oder Import)" + +#: mixins.py:225 +#, python-format +msgid "Something went wrong: %s" +msgstr "Etwas ist schief gelaufen: %s" + +#: mixins.py:249 mixins.py:295 +msgid "GitHub callback error" +msgstr "GitHub-Callback-Fehler" + +#: mixins.py:250 +msgid "State parameter did not match." +msgstr "Der Parameter 'state' stimmt nicht überein." + +#: mixins.py:296 +msgid "No redirect could be found." +msgstr "Kein redirect gefunden." + +#: mixins.py:438 +msgid "Authorize App" +msgstr "App autorisieren" + +#: mixins.py:439 +msgid "" +"To connect to GitHub repositories, you first need to authorize the MPDL app." +msgstr "" +"Die Verbindung zu GitHub-Repositorien erfordert die Autorisierung der MPDL-" +"App." + +#: mixins.py:447 +msgid "Install App" +msgstr "App installieren" + +#: mixins.py:448 +msgid "" +"To connect to GitHub repositories, you first need to install the MPDL app." +msgstr "" +"Die Verbindung zu GitHub-Repositorien erfordert die Installation der MPDL-" +"App." + +#: mixins.py:453 +msgid "Update list" +msgstr "Liste aktualisieren" + +#: mixins.py:454 +msgid "" +"List of your accessible GitHub repositories (up to 10 will be shown here)." +msgstr "" +"Liste der für Sie zugänglichen GitHub-Repositorien (bis zu 10 werden hier " +"angezeigt)." + +#: mixins.py:463 +msgid "You do not have any GitHub repositories yet" +msgstr "Sie haben noch keine GitHub-Repositorien" + +#: mixins.py:466 +msgid "To add more repositories to this list, click" +msgstr "Um die Repositorien-Liste zu erweitern, klicken Sie" + +#: mixins.py:467 +msgid "here" +msgstr "hier" + +#: mixins.py:473 +msgid "These are your most recently updated, accessible GitHub repositories." +msgstr "" +"Das sind Ihre zuletzt aktualisierten, zugänglichen Github-Repositorien." + +#: providers/exports.py:42 +msgid "CSV (comma separated)" +msgstr "CSV (Komma getrennt)" + +#: providers/exports.py:45 +msgid "CSV (semicolon separated)" +msgstr "CSV (Semikolon getrennt)" + +#: providers/exports.py:72 providers/exports.py:77 +msgid "example_folder/example_csv_file.csv" +msgstr "beispiel_ordner/beispiel_csv_datei.csv" + +#: providers/exports.py:82 +msgid "example_folder/example_json_file.json" +msgstr "beispiel_ordner/beispiel_json_datei.json" + +#: providers/exports.py:154 +msgid "Something went wrong" +msgstr "Etwas ist schief gelaufen" + +#: providers/exports.py:156 +msgid "" +"Either the export choices could not be created or the repository content " +"would have been overwritten without a warning." +msgstr "" +"Entweder konnten die Export-Optionen nicht erstellt werden oder der Inhalt " +"des Repositoriums wäre ohne Warnung überschrieben worden." + +#: providers/exports.py:210 +msgid "" +"A file with the same path exists in the selected repository and will be " +"overwritten" +msgstr "" +"Eine Datei mit demselben Pfad existiert im ausgewählten Repositorium und " +"wird überschrieben" + +#: providers/exports.py:308 +msgid "" +"not exported - it would have overwritten existing file in repository without " +"a warning." +msgstr "" +"nicht exportiert - es hätte die bestehende Datei im Repositorium ohne " +"Warnung überschrieben." + +#: providers/exports.py:322 +msgid "not exported - it could not be created." +msgstr "nicht exportiert - es konnte nicht erstellt werden." + +#: providers/exports.py:325 +msgid "successfully exported." +msgstr "erfolgreich exportiert." + +#: providers/exports.py:382 providers/exports.py:386 +msgid "not exported - something went wrong." +msgstr "nicht exportiert - etwas ist schief gelaufen." + +#: providers/exports.py:435 +msgid "Add GitHub integration" +msgstr "GitHub-Integration hinzufügen" + +#: providers/exports.py:436 +msgid "Send to GitHub" +msgstr "An GitHub senden" + +#: providers/exports.py:437 +msgid "" +"This integration allows the creation of issues in arbitrary GitHub " +"repositories. The upload of attachments is not supported by GitHub." +msgstr "" +"Diese Integration ermöglicht die Erstellung von Issues in beliebigen GitHub-" +"Repositorien. Das Hochladen von Anhängen wird von GitHub nicht unterstützt." + +#: providers/exports.py:444 +msgid "The URL of the GitHub repository to send issues to." +msgstr "" +"Die URL des GitHub-Repositoriums, an das Issues gesendet werden sollen." + +#: providers/exports.py:449 +msgid "The secret for a GitHub webhook to close a task (optional)." +msgstr "" +"Das 'secret' für einen GitHub-Webhook zum Schließen einer Aufgabe (optional)." + +#: providers/imports.py:105 +msgid "GitHub Import" +msgstr "GitHub-Import" + +#: providers/imports.py:119 providers/imports.py:169 +msgid "Import error" +msgstr "Fehler beim Import" + +#: providers/imports.py:120 +msgid "Something went wrong." +msgstr "Etwas ist schief gelaufen." + +#: providers/imports.py:170 +msgid "Either none of the import choices exist or they could not be requested." +msgstr "" +"Entweder existieren die Importoptionen nicht oder sie konnten nicht " +"angefragt werden." + +#: providers/imports.py:227 +msgid "" +"Either there is no file with this path in the selected repository or it " +"cannot be requested" +msgstr "" +"Entweder existiert keine Datei mit diesem Pfad im ausgewälten Repositorium " +"oder sie kann nicht angefragt werden" + +#: providers/imports.py:230 +msgid "Repository endpoint cannot be requested" +msgstr "Der Repositorium-Endpunkt kann nicht angefragt werden" + +#: providers/imports.py:336 +msgid " and" +msgstr " und" + +#: templates/plugins/github_export_form.html:10 +#: templates/plugins/github_export_form.html:12 +msgid "Export to GitHub" +msgstr "Nach GitHub exportieren" + +#: templates/plugins/github_export_success.html:6 +msgid "Successful Export to GitHub" +msgstr "Erfolgreicher Export nach GitHub" + +#: templates/plugins/github_export_success.html:14 +msgid "Go to GitHub repository" +msgstr "Zum GitHub-Repositorium" + +#: templates/plugins/github_import_success.html:12 +msgid "Successful Import from GitHub" +msgstr "Erfolgreicher Import aus GitHub" + +#: templates/plugins/github_import_success.html:15 +msgid "This import choice could not be requested" +msgstr "Diese Importoption konnte nicht angefragt werden" + +#: templates/plugins/github_import_success.html:17 +msgid "These import choices could not be requested" +msgstr "Diese Importoptionen konnten nicht angefragt werden" + +#: templates/plugins/github_import_success.html:25 +msgid "" +"You could either import the rest of the values found in the repository or " +"cancel the process." +msgstr "" +"Sie können den Rest der im Repositorium gefundenen Werte importieren oder " +"den Prozess abbrechen." + +#: templates/plugins/github_import_success.html:28 +msgid "Import anyway" +msgstr "Trotzdem importieren" + +#: templates/plugins/github_import_success.html:29 +msgid "Cancel" +msgstr "Abbrechen" diff --git a/rdmo_github/mixins.py b/rdmo_github/mixins.py index 5d8174d..800482f 100644 --- a/rdmo_github/mixins.py +++ b/rdmo_github/mixins.py @@ -1,14 +1,37 @@ +import logging +from urllib.parse import parse_qs, quote, urlencode, urlparse + from django.conf import settings +from django.http import HttpResponseRedirect +from django.shortcuts import render from django.urls import reverse +from django.utils.crypto import get_random_string +from django.utils.safestring import mark_safe +from django.utils.translation import gettext_lazy as _ + +import requests +from requests.auth import HTTPBasicAuth +from requests_toolbelt.multipart.encoder import MultipartEncoder +from rdmo.core.plugins import get_plugin from rdmo.services.providers import OauthProviderMixin +logger = logging.getLogger(__name__) + +APP_TYPE = settings.GITHUB_PROVIDER['app_type'] class GitHubProviderMixin(OauthProviderMixin): + install_url = f"https://github.com/apps/{settings.GITHUB_PROVIDER['github_app_name']}/installations/new" authorize_url = 'https://github.com/login/oauth/authorize' token_url = 'https://github.com/login/oauth/access_token' api_url = 'https://api.github.com' + PROVIDER_TYPES = [ + 'PROJECT_ISSUE_PROVIDERS', + 'PROJECT_EXPORTS', + 'PROJECT_IMPORTS' + ] + @property def client_id(self): return settings.GITHUB_PROVIDER['client_id'] @@ -21,10 +44,16 @@ def client_secret(self): def redirect_path(self): return reverse('oauth_callback', args=['github']) + def get_install_params(self, state): + return { + 'client_id': self.client_id, + 'state': state + } + def get_authorization_headers(self, access_token): return { - 'Authorization': f'token {access_token}', - 'Accept': 'application/vnd.github.v3+json' + 'Authorization': f'Bearer {access_token}', + 'Accept': 'application/vnd.github+json' } def get_authorize_params(self, request, state): @@ -43,5 +72,425 @@ def get_callback_params(self, request): 'code': request.GET.get('code') } + def get_validate_headers(self): + return { + 'Accept': 'application/vnd.github+json' + } + + def get_validate_params(self, access_token): + return {'access_token': access_token} + + def get_refresh_token_params(self, refresh_token): + return { + 'client_id': self.client_id, + 'client_secret': self.client_secret, + 'grant_type': 'refresh_token', + 'refresh_token': refresh_token + } + def get_error_message(self, response): return response.json().get('message') + + def get_state(self, request): + state = get_random_string(length=32) + self.store_in_session(request, 'state', state) + return state + + def get_app_config_url(self, request, installation_id): + if installation_id is None: + return None + + # get random state and store in session + state = self.get_state(request) + url = f'https://github.com/settings/installations/{installation_id}' + '?' + urlencode({'state': state}) + + return url + + def get_app_install_url(self, request): + # get random state and store in session + state = self.get_state(request) + url = self.install_url + '?' + urlencode(self.get_install_params(state)) + + return url + + def get_app_authorize_url(self, request): + # get random state and store in session + state = self.get_state(request) + url = self.authorize_url + '?' + urlencode(self.get_authorize_params(request, state)) + + return url + + def get_request_url(self, repo, path=None, suffix=None, ref=None): + url = '{api_url}/repos/{repo}'.format( + api_url=self.api_url, + repo=repo.replace('https://github.com/', '').strip('/') + ) + + if path: + url += '/contents/{path}'.format( + path=quote(path.removeprefix('../').removeprefix('./').strip('/'), safe='') + ) + + if suffix: + url += suffix + + if ref: + url += '?ref={ref}'.format(ref=quote(ref, safe='')) + + return url + + def process_app_context(self, request, *args, **kwargs): + # pop state from all github providers + for provider_type in self.PROVIDER_TYPES: + provider = get_plugin(provider_type, 'github') + if provider: + provider.pop_from_session(request, 'state') + + # save values in session + for k,v in kwargs.items(): + self.store_in_session(request, k, v) + + def make_request(self, request, method, url, apply_data_processing=False, *args, **kwargs): + methods = { + 'get': {'request_method': requests.get, 'success_method': self.get_success}, + 'post': {'request_method': requests.post, 'success_method': self.post_success}, + 'put': {'request_method': requests.put, 'success_method': self.put_success} + } + if method not in methods.keys(): + raise ValueError(f"Unsupported request method: {method}") + + access_token = self.get_from_session(request, 'access_token') + if access_token: + # if the access_token is available make request to the upstream service + logger.debug('%s: %s', method, url) + + request_method, success_method = methods[method].values() + + data_processing_params = {} + if 'data_processing_params' in kwargs.keys(): + data_processing_params = kwargs['data_processing_params'] + + headers = self.get_authorization_headers(access_token) + if 'multipart' in kwargs.keys(): + multipart = kwargs['multipart'] + if apply_data_processing: + processed_multipart = self.process_request_data( + {**multipart}, + access_token=access_token, + **data_processing_params + ) + multipart_encoder = MultipartEncoder(fields=processed_multipart) + else: + multipart_encoder = MultipartEncoder(fields=multipart) + + headers['Content-Type'] = multipart_encoder.content_type + response = request_method(url, data=multipart_encoder, headers=headers) + elif 'files' in kwargs.keys(): + files = kwargs['files'] + if apply_data_processing: + processed_files = self.process_request_data( + {**files}, + access_token=access_token, + **data_processing_params + ) + response = request_method(url, files=processed_files, headers=headers) + else: + response = request_method(url, files=files, headers=headers) + elif 'json' in kwargs.keys(): + json = kwargs['json'] + if apply_data_processing: + processed_json = self.process_request_data( + {**json}, + access_token=access_token, + **data_processing_params + ) + response = request_method(url, json=processed_json, headers=headers) + else: + response = request_method(url, json=json, headers=headers) + else: + response = request_method(url, headers=headers) + + if response.status_code == 401: + logger.warning('%s forbidden: %s (%s)', method, response.content, response.status_code) + else: + try: + response.raise_for_status() + return success_method(request, response) + + except requests.HTTPError: + logger.warning('%s error: %s (%s)', method, response.content, response.status_code) + + return render(request, 'core/error.html', { + 'title': _('Request (export or import) error'), + 'errors': [_('Something went wrong: %s') % self.get_error_message(response)] + }, status=200) + + # if the above did not work authorize first + kwargs['apply_data_processing'] = apply_data_processing + self.store_in_session(request, 'request', (method, url, kwargs)) + return self.authorize(request) + + def put_success(self, request, response): + raise NotImplementedError + + def authorize(self, request): + installation_id = self.get_from_session(request, 'installation_id') + if APP_TYPE == 'github_app' and installation_id is None: + url = self.get_app_install_url(request) + else: + url = self.get_app_authorize_url(request) + + return HttpResponseRedirect(url) + + def callback(self, request): + setup_action = request.GET.get('setup_action', None) + if setup_action != 'update' and request.GET.get('state') != self.pop_from_session(request, 'state'): + return render(request, 'core/error.html', { + 'title': _('GitHub callback error'), + 'errors': [_('State parameter did not match.')] + }, status=200) + + # store installation id of github app + installation_id = self.get_from_session(request, 'installation_id') + if APP_TYPE == 'github_app' and installation_id is None: + installation_id = request.GET.get('installation_id') + for provider_type in self.PROVIDER_TYPES: + provider = get_plugin(provider_type, 'github') + if provider: + provider.store_in_session(request, 'installation_id', installation_id) + + # authorization + access_token = self.validate_access_token(request, self.get_from_session(request, 'access_token')) + if access_token is None: + url = self.token_url + '?' + urlencode(self.get_callback_params(request)) + + response = requests.post(url, self.get_callback_data(request), + auth=self.get_callback_auth(request), + headers=self.get_callback_headers(request)) + + try: + response.raise_for_status() + except requests.HTTPError as e: + logger.error('callback authorization error: %s (%s)', response.content, response.status_code) + raise e + + response_data = response.json() + + access_token = response_data.get('access_token') + self.store_in_session(request, 'access_token', access_token) + self.store_in_session(request, 'refresh_token', response_data.get('refresh_token', None)) + + # After requesting new access_token or after github app installation or update + redirect_url = self.pop_from_session(request, 'redirect_url') + if redirect_url is not None: + return HttpResponseRedirect(redirect_url) + + try: + method, url, kwargs = self.pop_from_session(request, 'request') + return self.make_request(request, method, url, **kwargs) + except ValueError: + pass + + return render(request, 'core/error.html', { + 'title': _('GitHub callback error'), + 'errors': [_('No redirect could be found.')] + }, status=200) + + def get_validate_auth(self): + return HTTPBasicAuth(self.client_id, self.client_secret) + + def validate_access_token(self, request, access_token): + # https://docs.github.com/en/rest/apps/oauth-applications?apiVersion=2022-11-28#check-a-token + # https://docs.github.com/en/rest/authentication/authenticating-to-the-rest-api?apiVersion=2022-11-28#using-basic-authentication + + if access_token is None: + return None + + url = f'{self.api_url}/applications/{self.client_id}/token' + response = requests.post( + url, + headers=self.get_validate_headers(), + auth=self.get_validate_auth(), + json=self.get_validate_params(access_token)) + + try: + response.raise_for_status() + except requests.HTTPError: + access_token = self.refresh_access_token(request) + + return access_token + + def refresh_access_token(self, request): + 'Update access token with refresh_token if it exists' + + refresh_token = self.pop_from_session(request, 'refresh_token') + if refresh_token is None: + return None + + url = self.token_url + '?' + urlencode(self.get_refresh_token_params(refresh_token)) + response = requests.post(url, headers=self.get_validate_headers()) + + try: + response.raise_for_status() + response_error = response.json().get('error') + if response_error == 'bad_refresh_token': + self.pop_from_session(request, 'access_token') + return + except requests.HTTPError: + logger.error('GitHub refresh token error: %s (%s)', response.content, response.status_code) + return + + response_data = response.json() + + # store new access token in session + access_token = response_data.get('access_token') + self.store_in_session(request, 'access_token', access_token) + self.store_in_session(request, 'refresh_token', response_data.get('refresh_token')) + + return access_token + + def get_repo_choices(self, request, access_token, installation_id, minimum_repo_permission, page, per_page=10): + if access_token is None: + return [], False + + stored_repo_choices = self.get_from_session(request, 'github_repo_choices') + more_repos_available = self.get_from_session(request, 'github_more_repos_available') + more_repos_available = more_repos_available if more_repos_available is not None else True + + if stored_repo_choices and not more_repos_available: + return stored_repo_choices, more_repos_available + + if APP_TYPE == 'github_app': + # https://docs.github.com/de/rest/apps/installations?apiVersion=2022-11-28#list-repositories-accessible-to-the-user-access-token + url = f'{self.api_url}/user/installations/{installation_id}/repositories?per_page={per_page}' + else: + # https://docs.github.com/en/rest/repos/repos?apiVersion=2026-03-10#list-repositories-for-the-authenticated-user + url = '{api_url}/user/repos?per_page={per_page}&page={page}&sort={sort}'.format( + api_url=self.api_url, + per_page=per_page, + page=page, + sort='updated' + ) + + response = requests.get(url, headers=self.get_authorization_headers(access_token=access_token)) + try: + response.raise_for_status() + except requests.HTTPError: + logger.error('Error requesting GitHub repo list: %s (%s)', response.content, response.status_code) + return [], False + + if APP_TYPE == 'github_app': + repos = [ + r.get('html_url') for r in response.json().get('repositories', []) + if r.get('permissions', {}).get(minimum_repo_permission) + ] + total_repo_count = response.json().get('total_count') + else: + repos = [ + r.get('html_url') for r in response.json() + if r.get('permissions', {}).get(minimum_repo_permission) + ] + header_last_link = next( + ( + link.removesuffix('>; rel="last"').removeprefix('<') + for link in response.headers.get('Link', '').split(', ') + if link.endswith('; rel="last"') + ), + None + ) + parsed = urlparse(header_last_link) + query_params = parse_qs(parsed.query) + last_page = int(query_params.get('page')[0]) if query_params.get('page') else 0 + total_repo_count = last_page*per_page # max total_repo_count + + repo_choices = [(r, r) for r in repos] + + if stored_repo_choices: + repo_choices = stored_repo_choices + repo_choices + + more_repos_available = total_repo_count > page*per_page + + if APP_TYPE == 'oauth_app': + self.store_in_session(request, 'github_more_repos_available', more_repos_available) + self.store_in_session(request, 'github_repo_choices', repo_choices) + self.store_in_session(request, 'github_repos_page', page) + + return repo_choices, more_repos_available + + def get_repo_form_field_data(self, request, minimum_repo_permission): + access_token = self.validate_access_token(request, self.get_from_session(request, 'access_token')) + installation_id = self.get_from_session(request, 'installation_id') + + repos_page = self.pop_from_session(request, 'github_repos_page') + next_repos_page = repos_page + 1 if repos_page else 1 + repo_choices, more_repos_available = self.get_repo_choices( + request, + access_token, + installation_id, + minimum_repo_permission, + next_repos_page + ) + + app_actions = { + 'authorize': { + 'url_function': self.get_app_authorize_url, + 'url_kwargs': {'request': request}, + 'link_label': _('Authorize App'), + 'link_help_text': _('To connect to GitHub repositories, you first need to authorize the MPDL app.') + }, + } + if APP_TYPE == 'github_app': + app_actions.update({ + 'install': { + 'url_function': self.get_app_install_url, + 'url_kwargs': {'request': request}, + 'link_label': _('Install App'), + 'link_help_text': _('To connect to GitHub repositories, you first need to install the MPDL app.') + }, + 'update': { + 'url_function': self.get_app_config_url, + 'url_kwargs': {'request': request, 'installation_id': installation_id}, + 'link_label': _('Update list'), + 'link_help_text': _('List of your accessible GitHub repositories (up to 10 will be shown here).') + } + }) + # check if app was already (installed and) authorized, otherwise update repo access + action = 'install' if (APP_TYPE == 'github_app' and installation_id is None) else ( + 'authorize' if access_token is None else (None if APP_TYPE == 'oauth_app' else 'update') + ) + + if len(repo_choices) == 0: + repo_help_text = _('You do not have any GitHub repositories yet') + + elif action is None: + more_repos_link_text = _('To add more repositories to this list, click') + link_label = _('here') + more_repos_link = ( + f' {more_repos_link_text} {link_label}.' + if more_repos_available + else '' + ) + help_text = _('These are your most recently updated, accessible GitHub repositories.') + repo_help_text = mark_safe(f'{help_text} {more_repos_link}') + + else: + url_function, url_kwargs, link_label, link_help_text = app_actions[action].values() + url = url_function(**url_kwargs) + repo_help_text = mark_safe(f'{link_help_text} {link_label}') if url is not None else '' + + return repo_choices, repo_help_text + + + def get_form(self, request, form, *args, **kwargs): + repo_permission_map = { + 'GitHubExportForm': 'push', + 'GitHubImportForm': 'pull' + } + minimum_repo_permission = repo_permission_map[form.__name__] + repo_choices, repo_help_text = self.get_repo_form_field_data(request, minimum_repo_permission) + return form( + *args, + **kwargs, + repo_choices=repo_choices, + repo_help_text=repo_help_text + ) diff --git a/rdmo_github/providers.py b/rdmo_github/providers.py deleted file mode 100644 index c58e563..0000000 --- a/rdmo_github/providers.py +++ /dev/null @@ -1,137 +0,0 @@ -import base64 -import hmac -import json -from urllib.parse import quote - -from django import forms -from django.core.exceptions import ObjectDoesNotExist -from django.http import Http404, HttpResponse -from django.shortcuts import redirect, render -from django.utils.translation import gettext_lazy as _ - -from rdmo.core.imports import handle_fetched_file -from rdmo.projects.imports import RDMOXMLImport -from rdmo.projects.providers import OauthIssueProvider - -from .mixins import GitHubProviderMixin - - -class GitHubIssueProvider(GitHubProviderMixin, OauthIssueProvider): - add_label = _('Add GitHub integration') - send_label = _('Send to GitHub') - description = _('This integration allow the creation of issues in arbitrary GitHub repositories. ' - 'The upload of attachments is not supported by GitHub.') - - def get_post_url(self, request, issue, integration, subject, message, attachments): - repo_url = integration.get_option_value('repo_url') - if repo_url: - repo = repo_url.replace('https://github.com', '').strip('/') - return f'https://api.github.com/repos/{repo}/issues' - - def get_post_data(self, request, issue, integration, subject, message, attachments): - return { - 'title': subject, - 'body': message - } - - def get_issue_url(self, response): - return response.json().get('html_url') - - def webhook(self, request, integration): - secret = integration.get_option_value('secret') - header_signature = request.headers.get('X-Hub-Signature') - - if (secret is not None) and (header_signature is not None): - body_signature = 'sha1=' + hmac.new(secret.encode(), request.body, 'sha1').hexdigest() - - if hmac.compare_digest(header_signature, body_signature): - try: - payload = json.loads(request.body.decode()) - action = payload.get('action') - issue_url = payload.get('issue', {}).get('html_url') - - if action and issue_url: - try: - issue_resource = integration.resources.get(url=issue_url) - if action == 'closed': - issue_resource.issue.status = issue_resource.issue.ISSUE_STATUS_CLOSED - else: - issue_resource.issue.status = issue_resource.issue.ISSUE_STATUS_IN_PROGRESS - - issue_resource.issue.save() - except ObjectDoesNotExist: - pass - - return HttpResponse(status=200) - - except json.decoder.JSONDecodeError as e: - return HttpResponse(e, status=400) - - raise Http404 - - @property - def fields(self): - return [ - { - 'key': 'repo_url', - 'placeholder': 'https://github.com/username/repo', - 'help': _('The URL of the GitHub repository to send issues to.') - }, - { - 'key': 'secret', - 'placeholder': 'Secret (random) string', - 'help': _('The secret for a GitHub webhook to close a task (optional).'), - 'required': False, - 'secret': True - } - ] - - -class GitHubImport(GitHubProviderMixin, RDMOXMLImport): - - class Form(forms.Form): - repo = forms.CharField(label=_('GitHub repository'), - help_text=_('Please use the form username/repository or organization/repository.')) - path = forms.CharField(label=_('File path')) - ref = forms.CharField(label=_('Branch, tag, or commit'), initial='main') - - def render(self): - return render(self.request, 'projects/project_import_form.html', { - 'source_title': 'GitHub', - 'form': self.Form() - }, status=200) - - def submit(self): - form = self.Form(self.request.POST) - - if 'cancel' in self.request.POST: - if self.project is None: - return redirect('projects') - else: - return redirect('project', self.project.id) - - if form.is_valid(): - self.request.session['import_source_title'] = self.source_title = form.cleaned_data['path'] - - url = '{api_url}/repos/{repo}/contents/{path}?ref={ref}'.format( - api_url=self.api_url, - repo=quote(form.cleaned_data['repo']), - path=quote(form.cleaned_data['path']), - ref=quote(form.cleaned_data['ref']) - ) - - return self.get(self.request, url) - - return render(self.request, 'projects/project_import_form.html', { - 'source_title': 'GitHub', - 'form': form - }, status=200) - - def get_success(self, request, response): - file_content = response.json().get('content') - request.session['import_file_name'] = handle_fetched_file(base64.b64decode(file_content)) - - if self.current_project: - return redirect('project_update_import', self.current_project.id) - else: - return redirect('project_create_import') diff --git a/rdmo_github/providers/__init__.py b/rdmo_github/providers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/rdmo_github/providers/exports.py b/rdmo_github/providers/exports.py new file mode 100644 index 0000000..947518f --- /dev/null +++ b/rdmo_github/providers/exports.py @@ -0,0 +1,525 @@ +import base64 +import hmac +import json +import logging + +from django import forms +from django.core.exceptions import ObjectDoesNotExist +from django.http import Http404, HttpResponse +from django.shortcuts import redirect, render +from django.utils.translation import gettext +from django.utils.translation import gettext_lazy as _ + +import requests + +from rdmo_maus.exports.smp_exports import SMPExportMixin +from rdmo_maus.forms.validators import FilePathExtensionValidator, validate_file_path + +from rdmo.core.plugins import get_plugin +from rdmo.projects.exports import Export +from rdmo.projects.providers import OauthIssueProvider + +from ..forms.forms import GitHubExportForm +from ..mixins import GitHubProviderMixin +from ..utils import ( + clear_all_project_values_with_record_ids, + clear_project_value_with_record_id, + get_record_id_from_project_value, + set_record_id_on_project_value, +) + +logger = logging.getLogger(__name__) + +class GitHubExportProvider(GitHubProviderMixin, Export, SMPExportMixin): + @property + def export_choices(self): + catalog = self.project.catalog.uri_path + catalog = catalog.lower() if isinstance(catalog, str) else 'project_export' + + export_choices = { # check MultivalueCheckboxMultipleChoiceField in rdmo_maus.forms.fields.py for details + 'choices': [ + (f'False,data/{catalog}.xml', ('RDMO XML', _('File path')), 'xml'), + (f'False,data/{catalog}_comma_separated.csv', (_('CSV (comma separated)'), _('File path')), 'csvcomma'), + ( + f'False,data/{catalog}_semicolon_separated.csv', + (_('CSV (semicolon separated)'), _('File path')), + 'csvsemicolon' + ), + (f'False,data/{catalog}.json', ('JSON', _('File path')), 'json') + ], + 'choice_validators': { + 'xml': { + 'text': [validate_file_path, FilePathExtensionValidator('.xml')] + }, + 'csvcomma': { + 'text': [validate_file_path, FilePathExtensionValidator('.csv')] + }, + 'csvsemicolon': { + 'text': [validate_file_path, FilePathExtensionValidator('.csv')] + }, + 'json': { + 'text': [validate_file_path, FilePathExtensionValidator('.json')] + } + }, + 'choice_attributes': { + 'xml': { + 'text': { + 'placeholder': _('example_folder/example_xml_file.xml'), + } + }, + 'csvcomma': { + 'text': { + 'placeholder': _('example_folder/example_csv_file.csv'), + } + }, + 'csvsemicolon': { + 'text': { + 'placeholder': _('example_folder/example_csv_file.csv'), + } + }, + 'json': { + 'text': { + 'placeholder': _('example_folder/example_json_file.json'), + } + } + } + } + + smp_export_choices = getattr(self, 'smp_export_choices', None) + if smp_export_choices: + smp_export_choices.get('choices', []).extend(export_choices.get('choices', [])) + export_choices['choices'] = smp_export_choices.get('choices', []) + export_choices['choice_validators'].update(smp_export_choices.get('choice_validators', {})) + export_choices['choice_attributes'].update(smp_export_choices.get('choice_attributes', {})) + + return export_choices + + def render(self): + self.pop_from_session(self.request, 'github_export_choice_warnings') + redirect_url = self.request.build_absolute_uri() + self.process_app_context(self.request, redirect_url=redirect_url) + + access_token = self.validate_access_token(self.request, self.get_from_session(self.request, 'access_token')) + if access_token is None: + return self.authorize(self.request) + + context = { + 'form': self.get_form( + self.request, + GitHubExportForm, + export_choices=self.export_choices + ) + } + return render(self.request, 'plugins/github_export_form.html', context, status=200) + + def submit(self): + if 'cancel' in self.request.POST: + self.pop_from_session(self.request, 'github_more_repos_available') + self.pop_from_session(self.request, 'github_repo_choices') + self.pop_from_session(self.request, 'github_repos_page') + + if self.project is None: + return redirect('projects') + else: + clear_all_project_values_with_record_ids(self.project) + return redirect('project', self.project.id) + + self.store_in_session(self.request, 'github_more_repos_available', False) + + form = self.get_form(self.request, GitHubExportForm, self.request.POST, export_choices=self.export_choices) + if form.is_valid(): + + # 1. Validate export choices: Check submitted file paths to warn user if repo files will be overwritten + export_choice_warnings = self.get_from_session(self.request, 'github_export_choice_warnings') + new_repo = form.cleaned_data.get('new_repo') + + if not new_repo and export_choice_warnings is None: + context, export_choice_warnings = self.validate_export_choices(form.cleaned_data) + + if len(export_choice_warnings) > 0: + return render(self.request, 'plugins/github_export_form.html', context, status=200) + + # 2. Create file content for selected choices and export them + request_data, repo_html_url = self.process_form_data(form.cleaned_data) + + if repo_html_url is not None: + self.store_in_session(self.request, 'github_export_repo', repo_html_url) + + if request_data: + url = request_data[0].pop('url') + if len(request_data) > 1: + self.store_in_session(self.request, 'github_export_data', request_data[1:]) + else: + return render(self.request, 'core/error.html', { + 'title': _('Something went wrong'), + 'errors': [_( + 'Either the export choices could not be created or ' + 'the repository content would have been overwritten without a warning.' + )] + }, status=200) + + if new_repo: + return self.make_request(self.request, 'post', url, json=request_data[0]) + else: + return self.make_request(self.request, 'put', url, json=request_data[0]) + + new_repo = True if 'new_repo' in form.data else False + context = {'form': form} + return render(self.request, 'plugins/github_export_form.html', context, status=200) + + def validate_sha(self, project, export_choice, url, access_token): + """Validate the Github sha stored in the project.""" + + # Retrieve sha from the project's stored values + stored_sha = get_record_id_from_project_value(project, export_choice) + + # Send a GET request to Github to validate the stored sha + response = requests.get(url, headers=self.get_authorization_headers(access_token)) + if response.status_code == 200: + github_sha = response.json().get('sha') + + if stored_sha != github_sha: + set_record_id_on_project_value(project, github_sha, export_choice) + logger.warning('GitHubExportProvider - Updating stored sha: stored value for export choice ' + f'"{export_choice}" does not match with corresponding sha from github.') + + return github_sha + + elif response.status_code == 404: + logger.error(f'GitHubExportProvider - No matching resource for export choice "{export_choice}" found ' + 'in Github, deleting stored sha if it exists') + # the export_choice does not exist in GitHub, delete the corresponding sha from the project.value.text + clear_project_value_with_record_id(project, export_choice) + else: + # Log any other unexpected response code + logger.error(f'GitHubExportProvider - Error validating sha for export choice "{export_choice}": ' + f'{response.status_code}') + + def check_file_paths(self, exports, repo, branch): + access_token = self.get_from_session(self.request, 'access_token') + export_choice_warnings = {} + choice_keys = [] + for e in exports: + choice_key, file_path = e.split(',') + choice_keys.append(choice_key) + url = self.get_request_url(repo, path=file_path, ref=branch) + + sha = self.validate_sha(self.project, choice_key, url, access_token) + if sha: + export_choice_warnings[choice_key] = [ + gettext('A file with the same path exists in the selected repository and will be overwritten') + ] + + return export_choice_warnings, choice_keys, exports, branch + + def validate_export_choices(self, form_data): + export_choice_warnings, selected_choice_keys, checked_export_choices, checked_branch = self.check_file_paths( + form_data.get('exports'), + form_data.get('repo'), + form_data.get('branch') + ) + + self.store_in_session(self.request, 'github_export_choice_warnings', export_choice_warnings) + self.store_in_session(self.request, 'github_checked_export_choices', checked_export_choices) + self.store_in_session(self.request, 'github_checked_branch', checked_branch) + + selected_choices = [c for c in self.export_choices.get('choices', []) if c[2] in selected_choice_keys] + form = self.get_form( + self.request, + GitHubExportForm, + self.request.POST, + export_choices={ + **self.export_choices, + 'choices': selected_choices, + 'choice_warnings': export_choice_warnings + } + ) + context = {'form': form} + return context, export_choice_warnings + + def render_export(self, choice_key): + smp_export_choice_keys = getattr(self, 'smp_export_choice_keys', None) + if smp_export_choice_keys and choice_key in smp_export_choice_keys: + response = self.render_smp_export(choice_key) + else: + export_plugin = get_plugin('PROJECT_EXPORTS', choice_key) + export_plugin.project = self.project + response = export_plugin.render() + + return response + + def render_export_content(self, choice_key): + response = self.render_export(choice_key) + try: + binary = response.content + base64_bytes_of_content = base64.b64encode(binary) + base64_string_of_content = base64_bytes_of_content.decode('utf-8') + choice_content = base64_string_of_content + except AttributeError: + logger.warning(f'GitHubExportProvider - No content created for {choice_key}') + choice_content = None + + return choice_content + + def process_form_data(self, form_data, update_without_warning=False): + self.pop_from_session(self.request, 'github_export_choice_warnings') + request_data = [] + + # REPO + new_repo = form_data.get('new_repo') + repo_html_url = None + if new_repo: + request_data.append({ + 'name': form_data.get('new_repo_name'), + 'message': form_data.get('commit_message'), + 'url': f'{self.api_url}/user/repos' + }) + + repo = 'repo_placeholder' + else: + repo = form_data['repo'].replace('https://github.com/', '').strip('/') + repo_html_url = f'https://github.com/{repo}' + + # EXPORT OPTIONS + checked_export_choices = self.pop_from_session(self.request, 'github_checked_export_choices') + checked_branch = self.pop_from_session(self.request, 'github_checked_branch') + branch = 'main' if new_repo else form_data['branch'] + + exports = form_data.get('exports') + processed_exports = [] + for e in exports: + choice_key, file_path = e.split(',') + initial_file_path = file_path if new_repo else next( + (exp.split(',')[1] for exp in checked_export_choices if exp.split(',')[0] == choice_key), + file_path + ) + initial_branch = 'main' if new_repo else checked_branch + + if file_path != initial_file_path or branch != initial_branch: + new_export_choice_warnings, __, ___, ____ = self.check_file_paths([e], form_data['repo'], branch) + if choice_key in new_export_choice_warnings.keys() and not update_without_warning: + processed_exports.append({ + 'key': choice_key, + 'label': next( + (c[1][0] for c in self.export_choices.get('choices', []) if c[2] == choice_key), + choice_key + ), + 'success': False, + 'processing_status': _('not exported - it would have overwritten existing file ' + 'in repository without a warning.') + }) + continue + + choice_request_data = {} + stored_sha = get_record_id_from_project_value(self.project, choice_key) + if stored_sha is not None: + choice_request_data['sha'] = stored_sha + clear_project_value_with_record_id(self.project, choice_key) + + content = self.render_export_content(choice_key) + if content is None: + success = False + processing_status = _('not exported - it could not be created.') + else: + success = True + processing_status = _('successfully exported.') + + choice_request_data.update({ + 'message': form_data.get('commit_message'), + 'content': content, + 'branch': branch, + 'url': self.get_request_url(repo, path=file_path), + 'choice_key': choice_key + }) + request_data.append(choice_request_data) + + choice_label = next( + (c[1][0] for c in self.export_choices.get('choices', []) if c[2] == choice_key), + choice_key + ) + processed_exports.append({ + 'key': choice_key, + 'label': choice_label, + 'success': success, + 'processing_status': processing_status + }) + + successfully_processed_exports = list(filter(lambda x: x['success'], processed_exports)) + if len(successfully_processed_exports) == 0: + logger.warning('GitHubExportProvider - No export content could be created for the ' + f'selected choices: {exports}.') + return None, None + + self.store_in_session(self.request, 'github_processed_exports', processed_exports) + + return request_data, repo_html_url + + def put_data(self, request, request_data, processed_exports): + access_token = self.get_from_session(request, 'access_token') + + for file_json in request_data: + url = file_json.pop('url') + choice_key = file_json.pop('choice_key') + response = requests.put(url, json=file_json, headers=self.get_authorization_headers(access_token)) + + try: + response.raise_for_status() + except requests.HTTPError: + logger.error(f'GitHubExportProvider - Error putting {choice_key} to github: ' + f'{response.content} ({response.status_code})') + choice_label = next( + (c[1][0] for c in self.export_choices.get('choices', []) if c[2] == choice_key), + choice_key + ) + index, status = next( + ((i, s) for i, s in enumerate(processed_exports) if s['key'] == choice_key), + ( + len(processed_exports), + { + 'key': choice_key, + 'label': choice_label, + 'success': False, + 'processing_status': _('not exported - something went wrong.') + } + ) + ) + status.update({'success': False, 'processing_status': _('not exported - something went wrong.')}) + processed_exports[index] = status + + return processed_exports + + def post_success(self, request, response): + repo = response.json().get('full_name') + repo_html_url = response.json().get('html_url') + + request_data = self.pop_from_session(request, 'github_export_data') + processed_exports = self.pop_from_session(request, 'github_processed_exports') + self.pop_from_session(request, 'github_more_repos_available') + self.pop_from_session(request, 'github_repo_choices') + self.pop_from_session(request, 'github_repos_page') + + if isinstance(request_data , list): + request_data = [ + {**file_json, 'url': file_json['url'].replace('repo_placeholder', repo)} + for file_json in request_data + ] + processed_exports = self.put_data(request, request_data, processed_exports) + + successful_exports = list(filter(lambda x: x['success'], processed_exports)) + if len(successful_exports) == len(processed_exports): + return redirect(repo_html_url) + + context = {'repo_html_url': repo_html_url, 'processed_exports': processed_exports} + return render(request, 'plugins/github_export_success.html', context, status=200) + + def put_success(self, request, response): + request_data = self.pop_from_session(request, 'github_export_data') + repo_html_url = self.pop_from_session(request, 'github_export_repo') + processed_exports = self.pop_from_session(request, 'github_processed_exports') + self.pop_from_session(request, 'github_more_repos_available') + self.pop_from_session(request, 'github_repo_choices') + self.pop_from_session(request, 'github_repos_page') + + if isinstance(request_data , list): + processed_exports = self.put_data(request, request_data, processed_exports) + + successful_exports = list(filter(lambda x: x['success'], processed_exports)) + if len(successful_exports) == len(processed_exports): + return redirect(repo_html_url) + + context = {'repo_html_url': repo_html_url, 'processed_exports': processed_exports} + return render(request, 'plugins/github_export_success.html', context, status=200) + + +class GitHubIssueProvider(GitHubProviderMixin, OauthIssueProvider): + add_label = _('Add GitHub integration') + send_label = _('Send to GitHub') + description = _('This integration allows the creation of issues in arbitrary GitHub repositories. ' + 'The upload of attachments is not supported by GitHub.') + + _fields = { + 'repo_url': { + 'key': 'repo_url', + 'placeholder': 'https://github.com/username/repo', + 'help': _('The URL of the GitHub repository to send issues to.') + }, + 'secret': { + 'key': 'secret', + 'placeholder': 'Secret (random) string', + 'help': _('The secret for a GitHub webhook to close a task (optional).'), + 'required': False, + 'secret': True + } + } + + def get_post_url(self, request, issue, integration, subject, message, attachments): + repo_url = integration.get_option_value('repo_url') + if repo_url: + repo = repo_url.replace('https://github.com', '').strip('/') + return f'{self.api_url}/repos/{repo}/issues' + + def get_post_data(self, request, issue, integration, subject, message, attachments): + return { + 'title': subject, + 'body': message + } + + def get_issue_url(self, response): + return response.json().get('html_url') + + def webhook(self, request, integration): + secret = integration.get_option_value('secret') + header_signature = request.headers.get('X-Hub-Signature') + + if (secret is not None) and (header_signature is not None): + body_signature = 'sha1=' + hmac.new(secret.encode(), request.body, 'sha1').hexdigest() + + if hmac.compare_digest(header_signature, body_signature): + try: + payload = json.loads(request.body.decode()) + action = payload.get('action') + issue_url = payload.get('issue', {}).get('html_url') + + if action and issue_url: + try: + issue_resource = integration.resources.get(url=issue_url) + if action == 'closed': + issue_resource.issue.status = issue_resource.issue.ISSUE_STATUS_CLOSED + else: + issue_resource.issue.status = issue_resource.issue.ISSUE_STATUS_IN_PROGRESS + + issue_resource.issue.save() + except ObjectDoesNotExist: + pass + + return HttpResponse(status=200) + + except json.decoder.JSONDecodeError as e: + return HttpResponse(e, status=400) + + raise Http404 + + @property + def fields(self): + return self._fields.values() + + @fields.setter + def fields(self, new_fields): + if isinstance(new_fields, dict): + for k, v in new_fields.items(): + self._fields[k] = v + + def integration_setup(self, request): + redirect_url = request.build_absolute_uri() + self.process_app_context(request, redirect_url=redirect_url) + mininum_repo_permission = 'triage' + repo_choices, repo_help_text = self.get_repo_form_field_data(request, mininum_repo_permission) + + github_app_repo_url = {**self._fields['repo_url']} + if repo_choices is not None: + github_app_repo_url['widget'] = forms.RadioSelect(choices=repo_choices) + + if repo_help_text is not None: + github_app_repo_url['help'] = repo_help_text + + self.fields = {'repo_url': github_app_repo_url} diff --git a/rdmo_github/providers/imports.py b/rdmo_github/providers/imports.py new file mode 100644 index 0000000..f3b5b38 --- /dev/null +++ b/rdmo_github/providers/imports.py @@ -0,0 +1,434 @@ +import base64 +import logging + +from django.shortcuts import redirect, render +from django.utils.translation import gettext +from django.utils.translation import gettext_lazy as _ + +import requests + +from rdmo_maus.imports.mixins import SMPRepoImportMixin + +from rdmo.core.imports import handle_fetched_file +from rdmo.projects.imports import RDMOXMLImport + +from ..forms.forms import GitHubImportForm +from ..mixins import GitHubProviderMixin + +logger = logging.getLogger(__name__) + +class GitHubImportProvider(GitHubProviderMixin, SMPRepoImportMixin): + @property + def import_choices(self): + smp_import_choices = getattr(self, 'smp_import_choices', None) + if smp_import_choices: + return smp_import_choices + + return {} + + def render(self): + self.pop_from_session(self.request, 'github_import_choice_warnings') + redirect_url = self.request.build_absolute_uri() + self.process_app_context(self.request, redirect_url=redirect_url) + + access_token = self.validate_access_token(self.request, self.get_from_session(self.request, 'access_token')) + if access_token is None: + return self.authorize(self.request) + + form_kwargs = {} + if len(self.import_choices) > 0: + form_kwargs['import_choices'] = self.import_choices + + context = { + 'source_title': 'GitHub', + 'form': self.get_form( + self.request, + GitHubImportForm, + **form_kwargs + ) + } + return render(self.request, 'plugins/github_import_form.html', context, status=200) + + def submit(self): + if 'cancel' in self.request.POST: + self.pop_from_session(self.request, 'github_more_repos_available') + self.pop_from_session(self.request, 'github_repo_choices') + self.pop_from_session(self.request, 'github_repos_page') + + if self.current_project is None: + return redirect('projects') + else: + return redirect('project', self.current_project.id) + + method = self.request.POST.get('method') + if method == 'import_repo_subset': + return getattr(self, method)() + + self.store_in_session(self.request, 'github_more_repos_available', False) + + return self.process_form_submission() + + def get_success(self, request, response): + self.pop_from_session(request, 'github_more_repos_available') + self.pop_from_session(request, 'github_repo_choices') + self.pop_from_session(request, 'github_repos_page') + + # XML file import + # Only an xml file was imported and no processing is needed + if len(self.import_choices) == 0: + file_content = response.json().get('content') + request.session['import_file_name'] = handle_fetched_file(base64.b64decode(file_content)) + + if self.current_project: + return redirect('project_update_import', self.current_project.id) + else: + return redirect('project_create_import') + + # Multiple-source imports + # user can select multiple import sources, so processing is needed + request_urls = self.pop_from_session(self.request, 'request_urls') + import_choice_warnings = self.pop_from_session(self.request, 'github_import_choice_warnings') + + failed_import_choices = [] + for c in self.import_choices.get('choices', []): + if isinstance(import_choice_warnings, dict) and c[2] in import_choice_warnings.keys(): + choice_label = c[1][0] if isinstance(c[1], tuple) else c[1] + failed_import_choices.append(choice_label) + + authorization_headers = self.get_authorization_headers(self.get_from_session(request, 'access_token')) + process_import = getattr(self, 'process_import', None) + + if callable(process_import): + default_project_title = ( + response.json().get('html_url').split('/')[-1] + if response.json().get("html_url") + else _('GitHub Import') + ) + kwargs = { + 'request': request, + 'headers': authorization_headers, + 'request_urls': request_urls, + 'import_choice_warnings': import_choice_warnings, + 'default_project_title': default_project_title, + 'import_success_template': 'plugins/github_import_success.html', + 'import_success_context': {'failed_import_choices': failed_import_choices} + } + return process_import(**kwargs) + + return render(request, 'core/error.html', { + 'title': _('Import error'), + 'errors': [_("Something went wrong.")] + }, status=200) + + def import_repo_subset(self): + if self.current_project: + return redirect('project_update_import', self.current_project.id) + else: + return redirect('project_create_import') + + def process_form_submission(self): + form_kwargs = {} + if len(self.import_choices) > 0: + form_kwargs['import_choices'] = self.import_choices + + form = self.get_form( + self.request, + GitHubImportForm, + self.request.POST, + **form_kwargs + ) + + if form.is_valid(): + self.request.session['import_source_title'] = self.source_title = 'GitHub' + + # XML file import + # Only an xml file was imported and no processing is needed + if len(self.import_choices) == 0: + xml_url = self.process_form_data(form.cleaned_data, xml_url_only=True) + self.make_request(self.request, 'get', xml_url) + + # Multiple-source imports + # user can select multiple import sources, so processing is needed + + # 1. Validate import choices: Check submitted file paths to warn user if repo files don't exist + import_choice_warnings = self.get_from_session(self.request, 'github_import_choice_warnings') + + if import_choice_warnings is None: + context, import_choice_warnings = self.validate_import_choices(form.cleaned_data) + + if len(import_choice_warnings) > 0: + return render(self.request, 'plugins/github_import_form.html', context, status=200) + + # 2. Import selected choices + urls, import_choice_warnings = self.process_form_data(form.cleaned_data) + + repo_url = urls.pop('repo') + + if len(urls) == 0: + return render(self.request, 'core/error.html', { + 'title': _('Import error'), + 'errors': [_('Either none of the import choices exist or they could not be requested.')] + }, status=200) + else: + self.store_in_session(self.request, 'request_urls', urls) + + if len(import_choice_warnings) > 0: + self.store_in_session(self.request, 'github_import_choice_warnings', import_choice_warnings) + + return self.make_request(self.request, 'get', repo_url) + + context = { + 'source_title': 'GitHub', + 'form': form + } + + return render(self.request, 'plugins/github_import_form.html', context, status=200) + + def check_urls(self, form_data): + other_repo_check = form_data.get('other_repo_check') + repo = form_data.get('other_repo') if other_repo_check else form_data.get('repo') + + imports = {} + for i in form_data.get('imports', []): + i_list = i.split(',') + key = i_list[0] + value = i_list[1] if len(i_list) > 1 else None + imports[key] = value + + urls = { + 'repo': self.get_request_url(repo), + 'sbom': self.get_request_url(repo, suffix='/dependency-graph/sbom'), # only in default branch + 'languages': self.get_request_url(repo, suffix='/languages'), # only in default branch + 'xml': ( + self.get_request_url(repo, path=imports.get('xml'), ref=form_data.get('ref')) + if 'xml' in imports + else None + ), + 'citation': ( + self.get_request_url(repo, path=imports.get('citation'), ref=form_data.get('ref')) + if 'citation' in imports + else None + ), + 'license': self.get_request_url(repo) # independent of branch, last changes to LICENSE + } + selected_urls = {k:urls.get(k) for k in ['repo', *imports.keys()]} + + access_token = self.get_from_session(self.request, 'access_token') + import_choice_warnings = {} + choice_keys = [] + for choice_key, url in selected_urls.items(): + choice_keys.append(choice_key) + response = requests.get(url, headers=self.get_authorization_headers(access_token)) + + try: + response.raise_for_status() + except requests.HTTPError: + warning = ( + gettext('Either there is no file with this path in the selected repository ' + 'or it cannot be requested') + if imports.get(choice_key) # i.e. if form value has a file path + else gettext('Repository endpoint cannot be requested') + ) + import_choice_warnings[choice_key] = [warning] + + return import_choice_warnings, choice_keys, selected_urls + + def validate_import_choices(self, form_data): + import_choice_warnings, selected_choice_keys, _checked_import_urls = self.check_urls(form_data) + + self.store_in_session(self.request, 'github_import_choice_warnings', import_choice_warnings) + + selected_choices = [c for c in self.import_choices.get('choices', []) if c[2] in selected_choice_keys] + + form_kwargs = {} + if len(self.import_choices) > 0: + form_kwargs['import_choices'] = { + **self.import_choices, + 'choices': selected_choices, + 'choice_warnings': import_choice_warnings + } + + form = self.get_form( + self.request, + GitHubImportForm, + self.request.POST, + **form_kwargs + ) + + context = { + 'source_title': 'GitHub', + 'form': form + } + return context, import_choice_warnings + + def process_form_data(self, form_data, xml_url_only=False): + # XML file import + # Only an xml file was imported and no processing is needed + if xml_url_only: + other_repo_check = form_data.get('other_repo_check') + repo = form_data.get('other_repo') if other_repo_check else form_data.get('repo') + xml_url = self.get_request_url(repo, path=form_data.get('imports'), ref=form_data.get('ref')) + + return xml_url + + # Multiple-source imports + # user can select multiple import sources, so processing is needed + self.pop_from_session(self.request, 'github_import_choice_warnings') + + new_choice_warnings, __, new_urls = self.check_urls(form_data) + + selected_urls = {} + for choice_key, url in new_urls.items(): + if choice_key not in new_choice_warnings.keys(): + selected_urls[choice_key] = url + + return selected_urls, new_choice_warnings + + def get_license(self, url, headers): + license_id = None + + response = requests.get(url, headers=headers) + try: + response.raise_for_status() + license_dict = response.json().get('license') + if license_dict: + license_id = license_dict.get('spdx_id') + except requests.HTTPError: + pass + + return license_id + + def get_languages(self, url, headers): + languages = [] + + response = requests.get(url, headers=headers) + try: + response.raise_for_status() + languages = response.json().keys() + except requests.HTTPError: + pass + + return languages + + def get_sbom(self, url, headers): + response = requests.get(url, headers=headers) + try: + response.raise_for_status() + sbom = response.json().get('sbom') + except requests.HTTPError: + return {'dependencies': None, 'dependency_licenses': None} + + dependencies_str = '' + dependency_licenses = {} + repo_package_name = sbom.get('name') + for d in sbom.get('packages', []): + name = d.get('name') + if name == repo_package_name: # repo itself is listed as package + continue + + dependencies_str += f'{name}\n' + + license = d.get('licenseConcluded') if 'licenseConcluded' in d else ( + d.get('licenseDeclared') if 'licenseDeclared' in d else None + ) + if isinstance(license, str): + license = license.split(' AND') + license = ','.join(license[:-1]) + _(' and') + license[-1] if len(license) > 1 else license[0] + + if license and license in dependency_licenses: + dependency_licenses[license].append(name) + elif license and license not in dependency_licenses: + dependency_licenses[license] = [name] + + if len(dependencies_str) == 0: + dependencies_str = None + + dependency_licenses_str = '' + if len(dependency_licenses) > 0: + for k, v in dependency_licenses.items(): + dependency_licenses_str += f'{k} ({", ".join(v)})\n' + + if len(dependency_licenses_str) == 0: + dependency_licenses_str = None + + return {'dependencies': dependencies_str, 'dependency_licenses': dependency_licenses_str} + + def get_citation(self, url, headers): + content = None + + response = requests.get(url, headers=headers) + try: + response.raise_for_status() + encoded_content = response.json().get('content') + decoded_bytes = base64.b64decode(encoded_content) + content = decoded_bytes.decode('utf-8') + except requests.HTTPError: + pass + + return content + + +class GitHubImport(GitHubProviderMixin, RDMOXMLImport): + + def render(self): + redirect_url = self.request.build_absolute_uri() + self.process_app_context(self.request, redirect_url=redirect_url) + + access_token = self.validate_access_token(self.request, self.get_from_session(self.request, 'access_token')) + if access_token is None: + return self.authorize(self.request) + + context = { + 'source_title': 'GitHub', + 'form': self.get_form(self.request, GitHubImportForm) + } + return render(self.request, 'plugins/github_import_form.html', context, status=200) + + def submit(self): + if 'cancel' in self.request.POST: + self.pop_from_session(self.request, 'github_more_repos_available') + self.pop_from_session(self.request, 'github_repo_choices') + self.pop_from_session(self.request, 'github_repos_page') + + if self.current_project is None: + return redirect('projects') + else: + return redirect('project', self.current_project.id) + + self.store_in_session(self.request, 'github_more_repos_available', False) + + form = self.get_form(self.request, GitHubImportForm, self.request.POST) + if form.is_valid(): + self.request.session['import_source_title'] = self.source_title = form.cleaned_data['path'] + + url = self.process_form_data(form.cleaned_data) + return self.make_request(self.request, 'get', url) + + context = { + 'source_title': 'GitHub', + 'form': form + } + return render(self.request, 'plugins/github_import_form.html', context, status=200) + + def process_form_data(self, form_data): + other_repo_check = form_data.get('other_repo_check') + if other_repo_check: + repo = form_data.get('other_repo') + else: + repo = form_data.get('repo') + + url = self.get_request_url(repo, path=form_data.get('imports'), ref=form_data.get('ref')) + return url + + def get_success(self, request, response): + file_content = response.json().get('content') + request.session['import_file_name'] = handle_fetched_file(base64.b64decode(file_content)) + + self.pop_from_session(request, 'github_more_repos_available') + self.pop_from_session(request, 'github_repo_choices') + self.pop_from_session(request, 'github_repos_page') + + if self.current_project: + return redirect('project_update_import', self.current_project.id) + else: + return redirect('project_create_import') diff --git a/rdmo_github/static/plugins/js/github_form.js b/rdmo_github/static/plugins/js/github_form.js new file mode 100644 index 0000000..23e38a4 --- /dev/null +++ b/rdmo_github/static/plugins/js/github_form.js @@ -0,0 +1,36 @@ +const cbId = document.currentScript.getAttribute('cbId') +const checkedClass = document.currentScript.getAttribute('checkedClass') +const uncheckedClass = document.currentScript.getAttribute('uncheckedClass') + +document.addEventListener('DOMContentLoaded', () => { + if (cbId && checkedClass && uncheckedClass) { + toggleRepoFields(cbId, checkedClass, uncheckedClass) + } +}) + +function toggleRepoFields(cbId, checkedClass, uncheckedClass) { + const checkBox = document.getElementById(cbId) + let checkedCollection = document.getElementsByClassName(checkedClass) + let uncheckedCollection = document.getElementsByClassName(uncheckedClass) + + if (checkBox.checked == true){ + checkedCollection[0].style.display = 'block' + uncheckedCollection[0].style.display = 'none' + } else { + checkedCollection[0].style.display = 'none' + uncheckedCollection[0].style.display = 'block' + } +} + +function hideAllChoiceWarningMessages(text, choiceCount) { + let duration = 1000 + clearTimeout(text._timer) + text._timer = setTimeout(()=>{ + for (let i=0; i{% trans 'Export to GitHub' %} + + {% bootstrap_form submit=_('Export to GitHub') %} + +{% endblock %} \ No newline at end of file diff --git a/rdmo_github/templates/plugins/github_export_success.html b/rdmo_github/templates/plugins/github_export_success.html new file mode 100644 index 0000000..4f289c5 --- /dev/null +++ b/rdmo_github/templates/plugins/github_export_success.html @@ -0,0 +1,16 @@ +{% extends 'core/page.html' %} +{% load i18n core_tags %} + +{% block page %} + +

{% trans 'Successful Export to GitHub' %}

+ +
    + {% for export in processed_exports %} +
  • {{ export.label }}: {{ export.processing_status }}
    • + {% endfor %} +
+ + {% trans 'Go to GitHub repository' %} + +{% endblock %} \ No newline at end of file diff --git a/rdmo_github/templates/plugins/github_import_form.html b/rdmo_github/templates/plugins/github_import_form.html new file mode 100644 index 0000000..2cafef6 --- /dev/null +++ b/rdmo_github/templates/plugins/github_import_form.html @@ -0,0 +1,10 @@ +{% extends 'projects/project_import_form.html' %} +{% load i18n core_tags %} + +{% block head %} + {{ form.media }} +{% endblock %} + +{% block page %} + {{ block.super }} +{% endblock %} \ No newline at end of file diff --git a/rdmo_github/templates/plugins/github_import_success.html b/rdmo_github/templates/plugins/github_import_success.html new file mode 100644 index 0000000..9463832 --- /dev/null +++ b/rdmo_github/templates/plugins/github_import_success.html @@ -0,0 +1,34 @@ +{% extends 'core/page.html' %} +{% load i18n core_tags %} + +{% block content %} + +
+
+
+ {% csrf_token %} + + +

{% trans 'Successful Import from GitHub' %}

+ + {% if failed_import_choices|length == 1 %} + {% trans 'This import choice could not be requested' %}: {{ failed_import_choices.0 }}

+ {% else %} + {% trans 'These import choices could not be requested' %}: +
    + {% for import_label in failed_import_choices %} +
  • {{ import_label }}
    • + {% endfor %} +
+ {% endif %} + + {% trans 'You could either import the rest of the values found in the repository or cancel the process.' %} + +

+ + +
+
+
+ +{% endblock %} \ No newline at end of file diff --git a/rdmo_github/utils.py b/rdmo_github/utils.py new file mode 100644 index 0000000..7860f18 --- /dev/null +++ b/rdmo_github/utils.py @@ -0,0 +1,55 @@ +import logging + +from rdmo.domain.models import Attribute +from rdmo.projects.models.value import Value + +logger = logging.getLogger(__name__) + +attribute_uri_prefix = "https://rdmo.mpdl.mpg.de/terms" +attribute_sha_uri_key_prefix = "project/metadata/publication/github/sha/" + +def get_project_value_with_record_id(project, export_format): + record_id_attribute, _created = Attribute.objects.get_or_create(uri_prefix=attribute_uri_prefix, + key=f'{attribute_sha_uri_key_prefix}{export_format}') + + project_sha_value = project.values.filter(attribute=record_id_attribute).first() + return project_sha_value, record_id_attribute + +def get_record_id_from_project_value(project, export_format): + project_sha_value, _record_id_attribute = get_project_value_with_record_id(project, export_format) + + if project_sha_value is not None: + return project_sha_value.text + else: + return None + +def set_record_id_on_project_value(project, record_id, export_format): + if project is None or record_id is None: + return + + project_sha_value, record_id_attribute = get_project_value_with_record_id(project, export_format) + + if project_sha_value is None: + # create the value with text and add it + value = Value(project=project, attribute=record_id_attribute, text=record_id) + value.save() + project.values.add(value) + elif project_sha_value.text != record_id: + # update and overwrite the value.text + project_sha_value.text = record_id + project_sha_value.save() + +def clear_project_value_with_record_id(project, export_format): + '''Delete project value with record_id if it exists''' + + project_sha_value, _record_id_attribute = get_project_value_with_record_id(project, export_format) + if project_sha_value is not None: + project_sha_value.delete() + +def clear_all_project_values_with_record_ids(project): + ''' Delete all project values with record_id ''' + + sha_attributes = Attribute.objects.filter( + uri__startswith=f'{attribute_uri_prefix}/domain/{attribute_sha_uri_key_prefix}' + ) + project.values.filter(attribute__in=sha_attributes).delete()