From 5dc76a5145b0836171af2416d10224cbf7446d4c Mon Sep 17 00:00:00 2001 From: Konstantin Emtsev Date: Tue, 15 Mar 2022 08:32:42 +0300 Subject: [PATCH 1/2] js version wip --- download_figma_files.js | 339 ++--------------------------------- download_figma_files_core.js | 332 ++++++++++++++++++++++++++++++++++ get_figma_files_list.js | 251 ++++++++++++++++++++++++++ get_figma_files_list_core.js | 298 ++++++++++++++++++++++++++++++ manage.js | 87 +++++++++ 5 files changed, 978 insertions(+), 329 deletions(-) create mode 100644 download_figma_files_core.js create mode 100644 get_figma_files_list.js create mode 100644 get_figma_files_list_core.js create mode 100644 manage.js diff --git a/download_figma_files.js b/download_figma_files.js index 79ebeb1..a9fb804 100755 --- a/download_figma_files.js +++ b/download_figma_files.js @@ -1,8 +1,6 @@ #!/usr/bin/node - // Command line arguments - let figmaLogin = ''; let figmaPassword = ''; let figmaFilesListFile = ''; @@ -11,14 +9,7 @@ let debugDir = './process/debug'; let figmaUrl = ''; let settingFile = ''; -const settings = { - downloadTimeout: 1800, - selectorTimeout: 1500, - navigationTimeout: 180000, - launchTimeout: 120000, - loginTimeout: 10000, - pageOpenTimeout: 10000 -} +const downloadFigmaFiles = require('./download_figma_files_core') process.argv.forEach((arg) => { const keyValue = arg.split("="); @@ -47,328 +38,18 @@ process.argv.forEach((arg) => { } }) - if(!figmaLogin || !figmaPassword || (!figmaFilesListFile && !figmaUrl)) { console.log('Usage: []'); process.exit(1); } -const fs = require('fs'); -const fse = require('fs-extra') - -// Creating the directory for debug purposes - -if(debugDir) { - if(!debugDir.endsWith('/')) { - debugDir += '/'; - } - fs.mkdirSync(debugDir, {recursive: true}); - - console.log('Working in debug mode; content and screenshots of the file pages will be saved to the directory: ' + debugDir + '\n'); -} - - -// Reading Figma files list file - -let figmaFilesList; - -if (figmaFilesListFile) { - fs.readFile(figmaFilesListFile, (err, data) => { - if(err) throw err; - figmaFilesList = JSON.parse(data); - }); -} -if (figmaUrl) { - figmaFilesList = [ - { - uri: figmaUrl - } - ] -} - -if (settingFile) { - let newSettings = {} - const data = fs.readFileSync(settingFile); - newSettings = JSON.parse(data); - - settings.downloadTimeout = newSettings.downloadTimeout ?? settings.downloadTimeout; - settings.selectorTimeout = newSettings.selectorTimeout ?? settings.selectorTimeout; - settings.navigationTimeout = newSettings.navigationTimeout ?? settings.navigationTimeout; - settings.launchTimeout = newSettings.launchTimeout ?? settings.launchTimeout; - settings.loginTimeout = newSettings.loginTimeout ?? settings.loginTimeout; - settings.pageOpenTimeout = newSettings.pageOpenTimeout ?? settings.pageOpenTimeout; -} - -console.log('settings: ', settings); - -// Login page settings -const loginPageUrl = 'https://www.figma.com/login'; -const usernameSelector = 'input[name="email"]'; -const passwordSelector = 'input[name="password"]'; -const buttonSelector = 'button[type="submit"]'; - - -// After-login page settings -const filebrowserSelector = '#filebrowser-loading-page'; - - -// File page settings -const filePageBaseUrl = 'https://www.figma.com/file/'; -const fullscreenFilenameSelector = '#fullscreen-filename'; - - -// Using Puppeteer - -const puppeteer = require('puppeteer'); - -(async() => { - // Launching Chrome - const browser = await puppeteer.launch({args: ['--no-sandbox', '--disable-setuid-sandbox'], timeout: settings.launchTimeout}); - const page = await browser.newPage(); - page.setDefaultNavigationTimeout(settings.navigationTimeout); - - try { - // Opening the login page - console.log('Opening the login page'); - await page.goto(loginPageUrl, {waitUntil: 'networkidle2'}); - - - // Filling the email field - console.log('Filling the email field'); - await page.focus(usernameSelector, {delay: 500}); - await page.keyboard.type(figmaLogin); - - - // Filling the password field - console.log('Filling the password field'); - await page.focus(passwordSelector, {delay: 500}); - await page.keyboard.type(figmaPassword); - - - // Clicking the submit button - console.log('Clicking the submit button'); - await page.click(buttonSelector, {delay: 500}); - console.log('Waiting for an after-login page opening'); - - - // Sleeping for 10 seconds - await sleep(settings.loginTimeout); - - - // Waiting for an after-login page loading - await page.waitForSelector(filebrowserSelector); - } catch (err) { - console.error(err); - process.exit(1); - } - - if (!downloadsBaseDir.endsWith('/')) { - downloadsBaseDir += '/' - } - - let tmpDownloadDir = `${downloadsBaseDir}tmp/`; - - for (let t = 1; t < 1000; t++) { - if (!fs.existsSync(tmpDownloadDir)) { - break; - } - tmpDownloadDir = `${downloadsBaseDir}tmp${t}/`; - } - fs.mkdirSync(tmpDownloadDir, {recursive: true}); - - console.log(`Tmp directory to save the file: ${tmpDownloadDir}`); - - // Processing Figma files - - for(let i = 0; i < figmaFilesList.length; i++) { - let filePage = "" - if (figmaFilesList[i].uri && figmaFilesList[i].uri.startsWith("http")) { - filePage = figmaFilesList[i].uri - } else if (figmaFilesList[i].uri) { - filePage = filePageBaseUrl + figmaFilesList[i].uri - if (!filePage.endsWith("/")) { - filePage += "/" - } - } else { - filePage = filePageBaseUrl + figmaFilesList[i].key + '/'; - } - - console.log('\nStarting to process new file (' + (i + 1) + '/' + figmaFilesList.length + '), url: ' + filePage); - - try { - // Opening a file page - console.log('Opening the file page: ' + filePage); - let filePageResponse = await page.goto(filePage, {waitUntil: 'networkidle2'}); - - - // Checking status code - console.log('Returned status: ' + filePageResponse.status()); - if(filePageResponse.status() !== 200) { - console.log('Skipping the file'); - continue; - } - // Sleeping for 10 seconds, waiting for a specific element in React-generated content - await sleep(settings.pageOpenTimeout); - await page.waitForSelector(fullscreenFilenameSelector); - - // Checking if the file is available to save locally - - let content = await page.content(); - - if(content.includes('="Viewers can\'t copy or share this file."')) { - console.log('This file is protected against saving locally and sharing. Skipping') - continue; - } - // Getting and validating page title - - const title = await page.title(); - if(!title.endsWith(' – Figma')) { - console.log(`Title format "${title}" seems to be unrecognized, skipping the file`) - await page.screenshot({path: debugDir + 'login_screenshot' + '.png', fullPage: true}); - continue; - } - const fileName = title.replace(' – Figma', '').replaceAll('/', '_').replaceAll('|', '_').replaceAll('"', '_'); - // Getting the local path of the directory for the file to download - - let downloadDir = downloadsBaseDir; - - if(!downloadDir.endsWith('/')) { - downloadDir += '/'; - } - - if(figmaFilesList[i].team) { - downloadDir += 'TEAM ' + figmaFilesList[i].team + '/'; - } - - if(figmaFilesList[i].project) { - downloadDir += 'PROJECT ' + figmaFilesList[i].project + '/'; - } - if (!figmaFilesList[i].team && !figmaFilesList[i].project) { - downloadDir += fileName + '/'; - } - - - - console.log(`Directory to save the file: ${downloadDir}`); - - fs.mkdirSync(downloadDir, {recursive: true}); - - // Set download behavior - await page._client.send('Page.setDownloadBehavior', {behavior: 'allow', downloadPath: tmpDownloadDir}); - - // Debug: making screenshot and saving the page content - if(debugDir) { - fs.writeFile(debugDir + (i + 1) + '_content' + '.html', content, () => {}); - await page.screenshot({path: debugDir + (i + 1) + '_screenshot' + '.png', fullPage: true}); - } - - - await page.evaluate(_ => { - const mainMenu = document.querySelector('div[data-tooltip="main-menu"]'); - const clickEvt = document.createEvent("MouseEvents"); - clickEvt.initEvent("mousedown", true, true); - mainMenu.dispatchEvent(clickEvt); - }); - - await sleep(200); - - let menuItemFileHandle = null; - try { - menuItemFileHandle = await page.waitForSelector('div[data-testid="dropdown-option-File"]', {timeout: settings.selectorTimeout}); - } catch (error) { - if(debugDir && !menuItemFileHandle) { - console.log('cannot open main menu') - fs.writeFile(debugDir + fileName + '_main_menu' + '.html', content, () => {}); - await page.screenshot({path: debugDir + fileName + '_main_menu_screenshot' + '.png', fullPage: true}); - } - throw error; - } - - const box = await menuItemFileHandle.boundingBox(); - await page.mouse.move(box.x + 5, box.y + 5); - - await sleep(200); - let submenuFileHandle = null - try { - submenuFileHandle = await page.waitForSelector('div[data-testid="dropdown-option-Save local copy…"]', {timeout: settings.selectorTimeout}); - } catch (error) { - try { - const newFileItem = await page.waitForSelector('div[data-testid="dropdown-option-New design file"]', {timeout: settings.selectorTimeout}); - if (newFileItem) { - console.log('cannot save copy export not allowed') - continue; - } - } catch (error) { - //ignore - } - - if(debugDir && !submenuFileHandle) { - console.log('cannot open file menu') - fs.writeFile(debugDir + fileName + '_file_menu' + '.html', content, () => {}); - await page.screenshot({path: debugDir + fileName + '_file_menu_screenshot' + '.png', fullPage: true}); - } - throw error; - } - - - const saveFileBox = await submenuFileHandle.boundingBox(); - await page.mouse.move(saveFileBox.x + 5, saveFileBox.y + 5); - await page.mouse.click(saveFileBox.x + 5, saveFileBox.y + 5); - - let downloadedCheckTries = settings.downloadTimeout; - - let downloaded = false; - for(let j = 0; downloadedCheckTries == 0 || j < downloadedCheckTries; j++) { - - await sleep(1000); - fs.readdir(tmpDownloadDir, (err, files) => { - let donwloadedFile = '' - if (files && files.length > 0) { - donwloadedFile = files[0] - } - - if (donwloadedFile.length) { - if (fs.existsSync(downloadDir + donwloadedFile)) { - const tmpFileName = donwloadedFile.lastIndexOf('.'); - const now = new Date().toISOString().substring(0, 19).replaceAll('T', '_').replaceAll(':','-'); - fse.moveSync(downloadDir + donwloadedFile, downloadDir + donwloadedFile.substring(0, tmpFileName) + '_' + now + donwloadedFile.substring(tmpFileName)) - } - fse.moveSync(tmpDownloadDir + donwloadedFile, downloadDir + donwloadedFile); - - downloaded = true; - } - }); - - if (downloaded) { - console.log('Download complete'); - break; - } - - if (j % 30 == 0) { - console.log(`waiting file to download for ${parseInt(j / 60)} min ${j % 60} sec.`) - } - - if(downloadedCheckTries > 0 && j === (downloadedCheckTries - 1)) { - console.log(`File ${title} is not downloaded during timeout`) - } - } - } catch (err) { - console.error(err); - continue; - } - } - fs.rmdirSync(tmpDownloadDir); - - // Closing Chromium - browser.close(); -})(); - - -// Sleep function - -function sleep(ms) { - return new Promise(resolve => { - setTimeout(resolve, ms); - }) -} +downloadFigmaFiles({ + figmaLogin, + figmaPassword, + figmaFilesListFile, + figmaUrl, + downloadsBaseDir, + debugDir, + settingFile, +}) \ No newline at end of file diff --git a/download_figma_files_core.js b/download_figma_files_core.js new file mode 100644 index 0000000..b40af70 --- /dev/null +++ b/download_figma_files_core.js @@ -0,0 +1,332 @@ + +const path = require('path'); +const fs = require('fs'); +const fse = require('fs-extra') +const puppeteer = require('puppeteer'); + +function sleep(ms) { + return new Promise(resolve => { + setTimeout(resolve, ms); + }) +} + +const settings = { + downloadTimeout: 1800, + selectorTimeout: 20000, + navigationTimeout: 180000, + launchTimeout: 120000, + loginTimeout: 10000, + pageOpenTimeout: 20000 +} + +const downloadFigmaFiles = async(config = {}) => { + + let { + figmaLogin, + figmaPassword, + figmaFilesListFile, + figmaUrl, + downloadsBaseDir, + debugDir, + settingFile, + } = config + + console.log('figmaFilesListFile', figmaFilesListFile) + + + if(!figmaLogin || !figmaPassword || (!figmaFilesListFile && !figmaUrl)) + { + console.log('Usage: []'); + process.exit(1); + } + + // Creating the directory for debug purposes + + if(debugDir) { + if(!debugDir.endsWith('/')) { + debugDir += '/'; + } + fs.mkdirSync(debugDir, {recursive: true}); + + console.log('Working in debug mode; content and screenshots of the file pages will be saved to the directory: ' + debugDir + '\n'); + } + + + // Reading Figma files list file + + if (figmaFilesListFile) { + fs.readFile(figmaFilesListFile, (err, data) => { + if(err) throw err; + figmaFilesList = JSON.parse(data); + }); + } + if (figmaUrl) { + figmaFilesList = [ + { + uri: figmaUrl + } + ] + } + + if (settingFile) { + let newSettings = {} + const data = fs.readFileSync(settingFile); + newSettings = JSON.parse(data); + + settings.downloadTimeout = newSettings.downloadTimeout ?? settings.downloadTimeout; + settings.selectorTimeout = newSettings.selectorTimeout ?? settings.selectorTimeout; + settings.navigationTimeout = newSettings.navigationTimeout ?? settings.navigationTimeout; + settings.launchTimeout = newSettings.launchTimeout ?? settings.launchTimeout; + settings.loginTimeout = newSettings.loginTimeout ?? settings.loginTimeout; + settings.pageOpenTimeout = newSettings.pageOpenTimeout ?? settings.pageOpenTimeout; + } + + console.log('settings: ', settings); + + // Login page settings + const loginPageUrl = 'https://www.figma.com/login'; + const usernameSelector = 'input[name="email"]'; + const passwordSelector = 'input[name="password"]'; + const buttonSelector = 'button[type="submit"]'; + + + // After-login page settings + const filebrowserSelector = '#filebrowser-loading-page'; + + + // File page settings + const filePageBaseUrl = 'https://www.figma.com/file/'; + const fullscreenFilenameSelector = '#fullscreen-filename'; + + + // Launching Chrome + const browser = await puppeteer.launch({args: ['--no-sandbox', '--disable-setuid-sandbox'], timeout: settings.launchTimeout}); + const page = await browser.newPage(); + page.setDefaultNavigationTimeout(settings.navigationTimeout); + + try { + // Opening the login page + console.log('Opening the login page'); + await page.goto(loginPageUrl, {waitUntil: 'networkidle2'}); + + + // Filling the email field + console.log('Filling the email field'); + await page.focus(usernameSelector, {delay: 500}); + await page.keyboard.type(figmaLogin); + + + // Filling the password field + console.log('Filling the password field'); + await page.focus(passwordSelector, {delay: 500}); + await page.keyboard.type(figmaPassword); + + await page.screenshot({path: debugDir + 'login_form_screenshot' + '.png', fullPage: true}); + + // Clicking the submit button + console.log('Clicking the submit button'); + await page.click(buttonSelector, {delay: 500}); + console.log('Waiting for an after-login page opening'); + + + // Sleeping for 10 seconds + await sleep(settings.loginTimeout); + + await page.screenshot({path: debugDir + 'login_form_after_screenshot' + '.png', fullPage: true}); + + // Waiting for an after-login page loading + await page.waitForSelector(filebrowserSelector); + } catch (err) { + console.error(err); + process.exit(1); + } + + // Processing Figma files + + for(let i = 0; i < figmaFilesList.length; i++) { + let filePage = "" + if (figmaFilesList[i].uri && figmaFilesList[i].uri.startsWith("http")) { + filePage = figmaFilesList[i].uri + } else if (figmaFilesList[i].uri) { + filePage = filePageBaseUrl + figmaFilesList[i].uri + if (!filePage.endsWith("/")) { + filePage += "/" + } + } else { + filePage = filePageBaseUrl + figmaFilesList[i].key + '/'; + } + + console.log('\nStarting to process new file (' + (i + 1) + '/' + figmaFilesList.length + '), url: ' + filePage); + + try { + // Opening a file page + console.log('Opening the file page: ' + filePage); + let filePageResponse = await page.goto(filePage, {waitUntil: 'networkidle2'}); + + + // Checking status code + console.log('Returned status: ' + filePageResponse.status()); + if(filePageResponse.status() !== 200) { + console.log('Skipping the file'); + continue; + } + // Sleeping for 10 seconds, waiting for a specific element in React-generated content + await sleep(settings.pageOpenTimeout); + await page.waitForSelector(fullscreenFilenameSelector); + + // Checking if the file is available to save locally + + let content = await page.content(); + + if(content.includes('="Viewers can\'t copy or share this file."')) { + console.log('This file is protected against saving locally and sharing. Skipping') + continue; + } + // Getting and validating page title + + const title = await page.title(); + if(!title.endsWith(' – Figma')) { + console.log(`Title format "${title}" seems to be unrecognized, skipping the file`) + await page.screenshot({path: debugDir + 'login_screenshot' + '.png', fullPage: true}); + continue; + } + const fileName = title.replace(' – Figma', '').replaceAll('/', '_').replaceAll('|', '_').replaceAll('"', '_'); + // Getting the local path of the directory for the file to download + + let downloadDir = path.resolve(downloadsBaseDir); + + if(!downloadDir.endsWith('/')) { + downloadDir += '/'; + } + + if(figmaFilesList[i].team) { + downloadDir += 'TEAM ' + figmaFilesList[i].team + '/'; + } + + if(figmaFilesList[i].project) { + downloadDir += 'PROJECT ' + figmaFilesList[i].project + '/'; + } + if (!figmaFilesList[i].team && !figmaFilesList[i].project) { + downloadDir += fileName + '/'; + } + + downloadDir = path.normalize(downloadDir) + + + if (fs.existsSync(downloadDir + fileName+'.fig')) { + const now = new Date().toISOString().substring(0, 19).replaceAll('T', '_').replaceAll(':','-'); + fse.moveSync(downloadDir + fileName+'.fig', downloadDir + fileName + '_' + now + '.fig') + } + + if (fs.existsSync(downloadDir + fileName+'.jam')) { + const now = new Date().toISOString().substring(0, 19).replaceAll('T', '_').replaceAll(':','-'); + fse.moveSync(downloadDir + fileName+'.jam', downloadDir + fileName + '_' + now + '.jam') + } + + + console.log(`Directory to save the file: ${downloadDir}${fileName}`); + + fs.mkdirSync(downloadDir, {recursive: true}); + console.log(`created`); + + + // Set download behavior + await page._client.send('Page.setDownloadBehavior', {behavior: 'allow', downloadPath: downloadDir}); + console.log(`Page.setDownloadBehavior done`); + + // Debug: making screenshot and saving the page content + if(debugDir) { + fs.writeFile(debugDir + (i + 1) + '_content' + '.html', content, () => {}); + await page.screenshot({path: debugDir + fileName + (i + 1) + '_screenshot' + '.png', fullPage: true}); + } + + await page.evaluate(_ => { + const mainMenu = document.querySelector('div[data-tooltip="main-menu"]'); + const clickEvt = document.createEvent("MouseEvents"); + clickEvt.initEvent("mousedown", true, true); + mainMenu.dispatchEvent(clickEvt); + }); + + await sleep(1000); + + + console.log(`looking for download button`); + + let menuItemFileHandle = null; + try { + menuItemFileHandle = await page.waitForSelector('div[data-testid="dropdown-option-File"]', {timeout: settings.selectorTimeout}); + } catch (error) { + if(debugDir && !menuItemFileHandle) { + console.log('cannot open main menu') + fs.writeFile(debugDir + fileName + '_main_menu' + '.html', content, () => {}); + await page.screenshot({path: debugDir + fileName + '_main_menu_screenshot' + '.png', fullPage: true}); + } + throw error; + } + + const box = await menuItemFileHandle.boundingBox(); + await page.mouse.move(box.x + 5, box.y + 5); + + await sleep(200); + let submenuFileHandle = null + try { + submenuFileHandle = await page.waitForSelector('div[data-testid="dropdown-option-Save local copy…"]', {timeout: settings.selectorTimeout}); + } catch (error) { + try { + const newFileItem = await page.waitForSelector('div[data-testid="dropdown-option-New design file"]', {timeout: settings.selectorTimeout}); + if (newFileItem) { + console.log('cannot save copy export not allowed') + continue; + } + } catch (error) { + //ignore + } + + if(debugDir && !submenuFileHandle) { + console.log('cannot open file menu') + fs.writeFile(debugDir + fileName + '_file_menu' + '.html', content, () => {}); + await page.screenshot({path: debugDir + fileName + '_file_menu_screenshot' + '.png', fullPage: true}); + } + throw error; + } + + const saveFileBox = await submenuFileHandle.boundingBox(); + await page.mouse.move(saveFileBox.x + 5, saveFileBox.y + 5); + await page.mouse.click(saveFileBox.x + 5, saveFileBox.y + 5); + + await page.screenshot({path: debugDir + fileName + '_after_savefile' + '.png', fullPage: true}); + + let downloadedCheckTries = settings.downloadTimeout; + + for(let j = 0; downloadedCheckTries == 0 || j < downloadedCheckTries; j++) { + await page.screenshot({path: debugDir + fileName + '_dl_try_' + j + '.png', fullPage: true}); + + await sleep(1000); + if (fs.existsSync(downloadDir + fileName+'.fig')) { + console.log('Download complete'); + break; + } + if (fs.existsSync(downloadDir + fileName+'.jam')) { + console.log('Download complete'); + break; + } + + if (j % 30 == 0) { + console.log(`waiting file to download for ${parseInt(j / 60)} min ${j % 60} sec.`) + } + + if(downloadedCheckTries > 0 && j === (downloadedCheckTries - 1)) { + console.log(`File ${downloadDir + fileName} is not downloaded during timeout`) + } + } + } catch (err) { + console.error(err); + continue; + } + } + + // Closing Chromium + browser.close(); +} + +module.exports = downloadFigmaFiles diff --git a/get_figma_files_list.js b/get_figma_files_list.js new file mode 100644 index 0000000..683f0e4 --- /dev/null +++ b/get_figma_files_list.js @@ -0,0 +1,251 @@ +#!/usr/bin/python3 -u + +import json +import os +import shutil +from datetime import datetime +from pathlib import Path +from urllib import request +from urllib.error import HTTPError +from yaml import load, Loader + + +class FigmaFilesListGetter(object): + def __init__(self, config_file_path: Path = Path('./config/get_figma_files_list.yml')): + with open(config_file_path) as config_file: + self.config = load(config_file, Loader) + + self.token_header = {'X-Figma-Token': self.config['access_token']} + + def _http_request( + self, + request_url: str, + request_method: str = 'GET', + request_headers: dict or None = None, + request_data: bytes or None = None + ) -> dict: + http_request = request.Request(request_url, method=request_method) + + if request_headers: + http_request.headers = request_headers + + if request_data: + http_request.data = request_data + + try: + with request.urlopen(http_request) as http_response: + response_status = http_response.getcode() + response_headers = http_response.info() + response_data = http_response.read() + + except HTTPError as http_response_not_ok: + response_status = http_response_not_ok.getcode() + response_headers = http_response_not_ok.info() + response_data = http_response_not_ok.read() + + return { + 'status': response_status, + 'headers': response_headers, + 'data': response_data + } + + def _get_team_projects(self, team_id: str) -> list: + api_request_url = f'https://api.figma.com/v1/teams/{team_id}/projects' + + print(f'Getting team projects, requesting URL: {api_request_url}') + + api_response = self._http_request( + api_request_url, 'GET', self.token_header) + api_response_data = json.loads(api_response['data'].decode('utf-8')) + + print(f'Status: {api_response["status"]}') + + if api_response['status'] != 200 or api_response_data.get('err', None): + print('Failed to perform API request') + return [] + + team_name = api_response_data.get('name', team_id) + team_projects = [] + + for project in api_response_data.get('projects', []): + team_projects.append( + { + 'id': project['id'], + 'team': team_name + } + ) + + return team_projects + + def _get_teams_projects(self) -> list: + print('Getting the projects of all teams listed in config, if any, removing duplicates') + + teams_projects = [] + + for team_id in list(set(self.config.get('teams', []))): + team_projects = self._get_team_projects(str(team_id)) + teams_projects.extend(team_projects) + + return teams_projects + + def _merge_projects(self, teams_projects: list) -> list: + print('Merging teams projects with projects listed in config, if any, removing duplicates') + + merged_projects = [] + seen_projects_ids = set() + + for project in teams_projects: + if project['id'] not in seen_projects_ids: + merged_projects.append(project) + seen_projects_ids.add(project['id']) + + for project_id in list(set(self.config.get('projects', []))): + project_id = str(project_id) + + if project_id not in seen_projects_ids: + merged_projects.append( + { + 'id': project_id, + 'team': '' + } + ) + + seen_projects_ids.add(project_id) + + return merged_projects + + def _get_project_files(self, project: dict) -> list: + api_request_url = f'https://api.figma.com/v1/projects/{project["id"]}/files' + today = datetime.today().strftime('%Y-%m-%d') + + print(f'Getting project files, requesting URL: {api_request_url}') + + api_response = self._http_request( + api_request_url, 'GET', self.token_header) + + api_response_data = json.loads(api_response['data'].decode('utf-8')) + + print(f'Status: {api_response["status"]}') + + if api_response['status'] != 200 or api_response_data.get('err', None): + print('Failed to perform API request') + return [] + + project_name = api_response_data.get('name', project['id']) + project_files = [] + + for file in api_response_data.get('files', []): + + file_name_to_check = f'./store/TEAM {project["team"]}/PROJECT {project_name}/{file["name"]}.fig' + time = 0 + if os.path.isfile(file_name_to_check): + time = os.path.getmtime( + f'./store/TEAM {project["team"]}/PROJECT {project_name}/{file["name"]}.fig') + updates = datetime.strptime( + file["last_modified"], "%Y-%m-%dT%H:%M:%SZ").timestamp() + print(f'time: {time}, updated: {updates}') + if time < updates: + print( + f'File TEAM {project["team"]}/PROJECT {project_name}/{file["name"]}, key {file["key"]}, last modified: {file["last_modified"]} -> adding to the list') + project_files.append( + { + 'key': file['key'], + 'project': project_name, + 'team': project['team'], + 'last_modified': file['last_modified'] + } + ) + if os.path.isfile(file_name_to_check): + if not os.path.exists(f'./store/{today}'): + os.makedirs(f'./store/{today}') + if not os.path.exists(f'./store/{today}/TEAM {project["team"]}'): + os.makedirs(f'./store/{today}/TEAM {project["team"]}') + if not os.path.exists(f'./store/{today}/TEAM {project["team"]}/PROJECT {project_name}/'): + os.makedirs( + f'./store/{today}/TEAM {project["team"]}/PROJECT {project_name}/') + shutil.copyfile( + file_name_to_check, f'./store/{today}/TEAM {project["team"]}/PROJECT {project_name}/{file["name"]}.fig') + os.remove(file_name_to_check) + else: + print( + f'File {project["team"]}/{project_name}/{file["name"]}, key {file["key"]}, last modified: {file["last_modified"]} not modifided') + + return project_files + + def _get_projects_files(self, projects: list) -> list: + print('Getting the files of all projects') + + projects_files = [] + + for project in projects: + project_files = self._get_project_files(project) + projects_files.extend(project_files) + + return projects_files + + def _merge_files(self, projects_files: list) -> list: + print('Merging projects files with files listed in config, if any, removing duplicates') + + merged_files = [] + seen_files_keys = set() + + for file in projects_files: + if file['key'] not in seen_files_keys: + merged_files.append(file) + seen_files_keys.add(file['key']) + + for file_key in list(set(self.config.get('files', []))): + file_key = str(file_key) + + if file_key not in seen_files_keys: + merged_files.append( + { + 'key': file_key, + 'project': '', + 'team': '' + } + ) + + seen_files_keys.add(file_key) + + return merged_files + + def perform(self) -> None: + teams_projects = self._get_teams_projects() + projects = self._merge_projects(teams_projects) + projects_files = self._get_projects_files(projects) + files = self._merge_files(projects_files) + output_file_path = Path(self.config.get( + 'output_file', './process/figma_files_list.json')) + output_limit = self.config.get('output_limit', 0) + + if not output_limit or len(files) <= output_limit: + print('Writing the single output file') + + with open(output_file_path, 'w', encoding='utf8') as output_file: + json.dump(files, output_file, ensure_ascii=False, indent=4) + + else: + print('Writing multiple output files with partial slices') + + parts_count = int((len(files) - 1) / output_limit) + 1 + + for part_number in range(1, parts_count + 1): + start = output_limit * (part_number - 1) + stop = output_limit * (part_number) + files_part = files[start:stop] + + partial_output_file_path = ( + output_file_path.parent / + f'{output_file_path.stem}_part_{part_number}{output_file_path.suffix}' + ) + + with open(partial_output_file_path, 'w', encoding='utf8') as partial_output_file: + json.dump(files_part, partial_output_file, + ensure_ascii=False, indent=4) + + print('Done') + + +if __name__ == "__main__": + FigmaFilesListGetter().perform() diff --git a/get_figma_files_list_core.js b/get_figma_files_list_core.js new file mode 100644 index 0000000..c140064 --- /dev/null +++ b/get_figma_files_list_core.js @@ -0,0 +1,298 @@ +// import json +const fs = require('fs') +const path = require('path') +const yaml = require('yaml') +const date = require('date-and-time') +const { XMLHttpRequest } = require('xmlhttprequest') + +// import shutil +// from datetime import datetime +// from pathlib import Path +// from urllib import request +// from urllib.error import HTTPError +// from yaml import load, Loader + +const fetch = (url, opts={})=>{ + console.log('fetch', url) + const { mode='GET', headers } = opts + return new Promise((resolve, reject)=>{ + var xhttp = new XMLHttpRequest(); + + xhttp.onreadystatechange = function() { + // console.log('onreadystatechange', this.readyState, this.status) + if (this.readyState == 4 && this.status == 200) { + // Typical action to be performed when the document is ready: + // document.getElementById("demo").innerHTML = xhttp.responseText; + // console.log('respo', xhttp.responseText) + resolve(xhttp.responseText) + } + }; + xhttp.open(mode, url, true); + headers.forEach(([k, v])=>{ + // console.log('kv', k, v) + xhttp.setRequestHeader(k, v) + }) + xhttp.send(); + }) +} + +class FigmaFilesListGetter{ + constructor(config_file_path = path.resolve('./config/get_figma_files_list.yml')){ + console.log('FigmaFilesListGetter cfg', config_file_path) + const config_file = fs.readFileSync(config_file_path, 'utf-8') + console.log('config_file', config_file) + this.config = yaml.parse(config_file) + + this.token_headers = [['X-Figma-Token', this.config['access_token']||'']] + } + + async get_team_projects(team_id){ + try{ + + const api_request_url = `https://api.figma.com/v1/teams/${team_id}/projects` + + console.log(`Getting team projects, requesting URL: ${api_request_url}`) + + const api_response = await fetch( + api_request_url, + { + mode:'GET', + headers: this.token_headers + }) + + +// console.log('api_response', api_response) +// + const api_response_data = JSON.parse(api_response) + console.log(api_response_data) + + const team_name = api_response_data.name||team_id + const team_projects = [] + + const { projects=[] } = api_response_data + + for(let i in projects){ + team_projects.push( + { + 'id': projects[i].id, + 'team': team_name + } + ) + } + + return team_projects + + } catch(err){ + console.error(err) + return '' + } + } + + async get_teams_projects(){ + console.log('Getting the projects of all teams listed in config, if any, removing duplicates') + + let teams_projects = [] + + const teams = this.config['teams']||[] + console.log('teams', teams) + for(let i in teams){ + const team_projects = await this.get_team_projects(teams[i]) + teams_projects = teams_projects.concat(team_projects) + } + + return teams_projects + } + + merge_projects(teams_projects){ + console.log('Merging teams projects with projects listed in config, if any, removing duplicates') + + const merged_projects = [] + const seen_projects_ids = [] + + for(let i in teams_projects){ + const project = teams_projects[i] + if(!seen_projects_ids.find(id=>id == project.id)){ + merged_projects.push(project) + seen_projects_ids.push(project.id) + } + } +// +// for(let project_id in list(set(self.config.get('projects', [])))){ +// project_id = str(project_id) +// +// if(project_id not in seen_projects_ids){ +// merged_projects.append( +// { +// 'id': project_id, +// 'team': '' +// } +// ) +// +// seen_projects_ids.add(project_id) +// } +// } + + return merged_projects + } + + async get_project_files(project){ + const api_request_url = `https://api.figma.com/v1/projects/${project["id"]}/files` + const today = date.format(new Date(), 'YY-MM-DD') + + console.log(`Getting project files, requesting URL: ${api_request_url}`, today) + + const api_response = await fetch( + api_request_url, + { + mode:'GET', + headers: this.token_headers + }) + + const api_response_data = JSON.parse(api_response) + + // console.log('api_response_data', api_response_data) + + // if(api_response['status'] != 200 or api_response_data.get('err', None)){ + // console.log('Failed to perform API request') + // return [] + // } + + const project_name = api_response_data.name||project.id + const project_files = [] + + const { files } = api_response_data + + for(let i in files){ + const file = files[i] + + const file_name_to_check = `./store/TEAM ${project["team"]}/PROJECT ${project_name}/${file["name"]}.fig` + // let time = 0 + // if(os.path.isfile(file_name_to_check)){ + // time = os.path.getmtime(file_name_to_check) + // } + // const updates = datetime.strptime( + // file["last_modified"], "%Y-%m-%dT%H:%M:%SZ").timestamp() + // console.log(f'time: {time}, updated: {updates}') + // if(time < updates){ + // console.log(`File TEAM ${project["team"]}/PROJECT ${project_name}/${file["name"]}, key ${file["key"]}, last modified: ${file["last_modified"]} -> adding to the list`) + project_files.push( + { + 'key': file.key, + 'name': file.name, + 'project': project_name, + 'team': project.team, + 'last_modified': file['last_modified'] + } + ) + // if(os.path.isfile(file_name_to_check)){ + // if not os.path.exists(f'./store/{today}'): + // os.makedirs(f'./store/{today}') + // if not os.path.exists(f'./store/{today}/TEAM {project["team"]}'): + // os.makedirs(f'./store/{today}/TEAM {project["team"]}') + // if not os.path.exists(f'./store/{today}/TEAM {project["team"]}/PROJECT {project_name}/'): + // os.makedirs( + // f'./store/{today}/TEAM {project["team"]}/PROJECT {project_name}/') + // shutil.copyfile( + // file_name_to_check, f'./store/{today}/TEAM {project["team"]}/PROJECT {project_name}/{file["name"]}.fig') + // os.remove(file_name_to_check) + // } + // }else{ + // console.log(`File ${project["team"]}/${project_name}/${file["name"]}, key ${file["key"]}, last modified: ${file["last_modified"]} not modifided`) + // } + } + + return project_files + } + + async get_projects_files(projects){ + console.log('Getting the files of all projects') + + let projects_files = [] + + for( let i in projects ){ + const project_files = await this.get_project_files(projects[i]) + projects_files = projects_files.concat(project_files) + } + + return projects_files + } + + merge_files(projects_files){ + console.log('Merging projects files with files listed in config, if any, removing duplicates') + + const merged_files = [] + const seen_files_keys = [] + + for(let i in projects_files){ + const file = projects_files[i] + if(!seen_files_keys.find(v=>v==file.key)){ + merged_files.push(file) + seen_files_keys.push(file.key) + } + } + + const file_keys = this.config.files||[] + for(let i in file_keys){ + const file_key = ""+file_keys[i] + + if(!seen_files_keys.find(v=>v==file_key)){ + merged_files.push( + { + 'key': file_key, + 'project': '', + 'team': '' + } + ) + + seen_files_keys.push(file_key) + } + } + + return merged_files + } + + async perform(){ + const teams_projects = await this.get_teams_projects() + // console.log('teams_projects', teams_projects) + const projects = this.merge_projects(teams_projects) + // console.log('projects', projects) + const projects_files = await this.get_projects_files(projects) + const files = this.merge_files(projects_files) + console.log('files', files) +// const output_file_path = Path(self.config.get( +// 'output_file', './process/figma_files_list.json')) +// const output_limit = self.config.get('output_limit', 0) +// +// if not output_limit or len(files) <= output_limit: +// console.log('Writing the single output file') +// +// with open(output_file_path, 'w', encoding='utf8') as output_file: +// json.dump(files, output_file, ensure_ascii=False, indent=4) +// +// else: +// console.log('Writing multiple output files with partial slices') +// +// parts_count = int((len(files) - 1) / output_limit) + 1 +// +// for part_number in range(1, parts_count + 1): +// start = output_limit * (part_number - 1) +// stop = output_limit * (part_number) +// files_part = files[start:stop] +// +// partial_output_file_path = ( +// output_file_path.parent / +// f'{output_file_path.stem}_part_{part_number}{output_file_path.suffix}' +// ) +// +// with open(partial_output_file_path, 'w', encoding='utf8') as partial_output_file: +// json.dump(files_part, partial_output_file, +// ensure_ascii=False, indent=4) +// +// console.log('Done') + + } +} +// if __name__ == "__main__": +// FigmaFilesListGetter().perform() + +module.exports = FigmaFilesListGetter \ No newline at end of file diff --git a/manage.js b/manage.js new file mode 100644 index 0000000..fdc570f --- /dev/null +++ b/manage.js @@ -0,0 +1,87 @@ +const process = require('process'); +const { spawn } = require('child_process') +const fs = require('fs/promises'); +const path = require('path'); +const downloadFigmaFiles = require('./download_figma_files_core') +const FigmaFilesListGetter = require('./get_figma_files_list_core') + +// ======== +// Settings +// ======== + +// Working directory +const working_dir_path= path.resolve('./process1') +const store_dir_path= path.resolve('./store1') + +const downloads_dir_name = path.normalize(working_dir_path + '/_downloads') +const get_list_log_name = path.normalize(working_dir_path + '/get_figma_files_list_log.txt') +const download_log_name = path.normalize(store_dir_path + '/download_figma_files_log.txt') +const partial_lists_names = path.normalize(working_dir_path + '/figma_files_list*.json') + +// Temporary directory +const tmp_dir_path = working_dir_path + +// HTTP server directory +const http_server_dir_path = './figma' +const deduplicated_dir_name = 'deduplicated' +const stat_log_name = 'stats_log.txt' +const archive_dirs_to_keep = 30 +const archive_dir_name_pattern = '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]' +const df_line_to_grep = '/dev/mapper/centos-root' + +const args = process.argv.slice(2) +console.log('args', args) + +// Other +const figma_login = args[0] +const figma_password = args[1] +// const current_date = `date +"%Y-%m-%d"` + +// +// if (!figma_login){ +// console.error("figma login needed") +// process.exit(1) +// } +// +// if (!figma_password){ +// console.error("figma password needed") +// process.exit(1) +// } + +// ======================== +// Getting files from Figma +// ======================== + +// Coming to the working directory +// Remove files that may be kept since previous session +; +(async()=>{ + + console.log('downloads_dir_name', working_dir_path) + await fs.rm(working_dir_path, { recursive:true, force:true }) + await fs.mkdir(working_dir_path, { recursive:true }) + await fs.mkdir(store_dir_path, { recursive:true }) + + const fileListGetter = new FigmaFilesListGetter() + fileListGetter.perform() +})() + +// . +// // Getting files list +// ./get_figma_files_list.py 2>&1 | tee ${get_list_log_name} +// +// // Downloading files +// +// list_names=`ls ${partial_lists_names}` +// + +// downloadFigmaFiles() + +// for list_name in ${list_names} ; do +// echo -e "\n\nProcessing the file ${list_name}\n\n" +// node ./download_figma_files.js "figmaLogin=${figma_login}" "figmaPassword=${figma_password}" "figmaFilesList=./${list_name}" "settingFile=./config/download_settings.json" 2>&1 +// cp -rf ./${downloads_dir_name}/* ${store_dir_path}/ +// sleep 30 +// done +// +// echo 'DONE' \ No newline at end of file From a01483ae08db3701e813bfdb77297c953eda875f Mon Sep 17 00:00:00 2001 From: Konstantin Emtsev Date: Tue, 15 Mar 2022 09:21:24 +0300 Subject: [PATCH 2/2] first working js version --- download_figma_files.js | 4 +- download_figma_files_core.js | 4 +- get_figma_files_list.js | 254 +---------------------------------- get_figma_files_list_core.js | 60 ++++----- manage.js | 74 +++++----- package.json | 5 +- 6 files changed, 77 insertions(+), 324 deletions(-) diff --git a/download_figma_files.js b/download_figma_files.js index a9fb804..0a7198a 100755 --- a/download_figma_files.js +++ b/download_figma_files.js @@ -4,8 +4,8 @@ let figmaLogin = ''; let figmaPassword = ''; let figmaFilesListFile = ''; -let downloadsBaseDir = './process/_downloads'; -let debugDir = './process/debug'; +let downloadsBaseDir = ''; +let debugDir = ''; let figmaUrl = ''; let settingFile = ''; diff --git a/download_figma_files_core.js b/download_figma_files_core.js index b40af70..8fc948a 100644 --- a/download_figma_files_core.js +++ b/download_figma_files_core.js @@ -26,8 +26,8 @@ const downloadFigmaFiles = async(config = {}) => { figmaPassword, figmaFilesListFile, figmaUrl, - downloadsBaseDir, - debugDir, + downloadsBaseDir = './process/_downloads', + debugDir = './process/debug', settingFile, } = config diff --git a/get_figma_files_list.js b/get_figma_files_list.js index 683f0e4..fe60741 100644 --- a/get_figma_files_list.js +++ b/get_figma_files_list.js @@ -1,251 +1,3 @@ -#!/usr/bin/python3 -u - -import json -import os -import shutil -from datetime import datetime -from pathlib import Path -from urllib import request -from urllib.error import HTTPError -from yaml import load, Loader - - -class FigmaFilesListGetter(object): - def __init__(self, config_file_path: Path = Path('./config/get_figma_files_list.yml')): - with open(config_file_path) as config_file: - self.config = load(config_file, Loader) - - self.token_header = {'X-Figma-Token': self.config['access_token']} - - def _http_request( - self, - request_url: str, - request_method: str = 'GET', - request_headers: dict or None = None, - request_data: bytes or None = None - ) -> dict: - http_request = request.Request(request_url, method=request_method) - - if request_headers: - http_request.headers = request_headers - - if request_data: - http_request.data = request_data - - try: - with request.urlopen(http_request) as http_response: - response_status = http_response.getcode() - response_headers = http_response.info() - response_data = http_response.read() - - except HTTPError as http_response_not_ok: - response_status = http_response_not_ok.getcode() - response_headers = http_response_not_ok.info() - response_data = http_response_not_ok.read() - - return { - 'status': response_status, - 'headers': response_headers, - 'data': response_data - } - - def _get_team_projects(self, team_id: str) -> list: - api_request_url = f'https://api.figma.com/v1/teams/{team_id}/projects' - - print(f'Getting team projects, requesting URL: {api_request_url}') - - api_response = self._http_request( - api_request_url, 'GET', self.token_header) - api_response_data = json.loads(api_response['data'].decode('utf-8')) - - print(f'Status: {api_response["status"]}') - - if api_response['status'] != 200 or api_response_data.get('err', None): - print('Failed to perform API request') - return [] - - team_name = api_response_data.get('name', team_id) - team_projects = [] - - for project in api_response_data.get('projects', []): - team_projects.append( - { - 'id': project['id'], - 'team': team_name - } - ) - - return team_projects - - def _get_teams_projects(self) -> list: - print('Getting the projects of all teams listed in config, if any, removing duplicates') - - teams_projects = [] - - for team_id in list(set(self.config.get('teams', []))): - team_projects = self._get_team_projects(str(team_id)) - teams_projects.extend(team_projects) - - return teams_projects - - def _merge_projects(self, teams_projects: list) -> list: - print('Merging teams projects with projects listed in config, if any, removing duplicates') - - merged_projects = [] - seen_projects_ids = set() - - for project in teams_projects: - if project['id'] not in seen_projects_ids: - merged_projects.append(project) - seen_projects_ids.add(project['id']) - - for project_id in list(set(self.config.get('projects', []))): - project_id = str(project_id) - - if project_id not in seen_projects_ids: - merged_projects.append( - { - 'id': project_id, - 'team': '' - } - ) - - seen_projects_ids.add(project_id) - - return merged_projects - - def _get_project_files(self, project: dict) -> list: - api_request_url = f'https://api.figma.com/v1/projects/{project["id"]}/files' - today = datetime.today().strftime('%Y-%m-%d') - - print(f'Getting project files, requesting URL: {api_request_url}') - - api_response = self._http_request( - api_request_url, 'GET', self.token_header) - - api_response_data = json.loads(api_response['data'].decode('utf-8')) - - print(f'Status: {api_response["status"]}') - - if api_response['status'] != 200 or api_response_data.get('err', None): - print('Failed to perform API request') - return [] - - project_name = api_response_data.get('name', project['id']) - project_files = [] - - for file in api_response_data.get('files', []): - - file_name_to_check = f'./store/TEAM {project["team"]}/PROJECT {project_name}/{file["name"]}.fig' - time = 0 - if os.path.isfile(file_name_to_check): - time = os.path.getmtime( - f'./store/TEAM {project["team"]}/PROJECT {project_name}/{file["name"]}.fig') - updates = datetime.strptime( - file["last_modified"], "%Y-%m-%dT%H:%M:%SZ").timestamp() - print(f'time: {time}, updated: {updates}') - if time < updates: - print( - f'File TEAM {project["team"]}/PROJECT {project_name}/{file["name"]}, key {file["key"]}, last modified: {file["last_modified"]} -> adding to the list') - project_files.append( - { - 'key': file['key'], - 'project': project_name, - 'team': project['team'], - 'last_modified': file['last_modified'] - } - ) - if os.path.isfile(file_name_to_check): - if not os.path.exists(f'./store/{today}'): - os.makedirs(f'./store/{today}') - if not os.path.exists(f'./store/{today}/TEAM {project["team"]}'): - os.makedirs(f'./store/{today}/TEAM {project["team"]}') - if not os.path.exists(f'./store/{today}/TEAM {project["team"]}/PROJECT {project_name}/'): - os.makedirs( - f'./store/{today}/TEAM {project["team"]}/PROJECT {project_name}/') - shutil.copyfile( - file_name_to_check, f'./store/{today}/TEAM {project["team"]}/PROJECT {project_name}/{file["name"]}.fig') - os.remove(file_name_to_check) - else: - print( - f'File {project["team"]}/{project_name}/{file["name"]}, key {file["key"]}, last modified: {file["last_modified"]} not modifided') - - return project_files - - def _get_projects_files(self, projects: list) -> list: - print('Getting the files of all projects') - - projects_files = [] - - for project in projects: - project_files = self._get_project_files(project) - projects_files.extend(project_files) - - return projects_files - - def _merge_files(self, projects_files: list) -> list: - print('Merging projects files with files listed in config, if any, removing duplicates') - - merged_files = [] - seen_files_keys = set() - - for file in projects_files: - if file['key'] not in seen_files_keys: - merged_files.append(file) - seen_files_keys.add(file['key']) - - for file_key in list(set(self.config.get('files', []))): - file_key = str(file_key) - - if file_key not in seen_files_keys: - merged_files.append( - { - 'key': file_key, - 'project': '', - 'team': '' - } - ) - - seen_files_keys.add(file_key) - - return merged_files - - def perform(self) -> None: - teams_projects = self._get_teams_projects() - projects = self._merge_projects(teams_projects) - projects_files = self._get_projects_files(projects) - files = self._merge_files(projects_files) - output_file_path = Path(self.config.get( - 'output_file', './process/figma_files_list.json')) - output_limit = self.config.get('output_limit', 0) - - if not output_limit or len(files) <= output_limit: - print('Writing the single output file') - - with open(output_file_path, 'w', encoding='utf8') as output_file: - json.dump(files, output_file, ensure_ascii=False, indent=4) - - else: - print('Writing multiple output files with partial slices') - - parts_count = int((len(files) - 1) / output_limit) + 1 - - for part_number in range(1, parts_count + 1): - start = output_limit * (part_number - 1) - stop = output_limit * (part_number) - files_part = files[start:stop] - - partial_output_file_path = ( - output_file_path.parent / - f'{output_file_path.stem}_part_{part_number}{output_file_path.suffix}' - ) - - with open(partial_output_file_path, 'w', encoding='utf8') as partial_output_file: - json.dump(files_part, partial_output_file, - ensure_ascii=False, indent=4) - - print('Done') - - -if __name__ == "__main__": - FigmaFilesListGetter().perform() +const FigmaFilesListGetter = require('./get_figma_files_list_core') +const fileListGetter = new FigmaFilesListGetter() +fileListGetter.perform() diff --git a/get_figma_files_list_core.js b/get_figma_files_list_core.js index c140064..b28eb90 100644 --- a/get_figma_files_list_core.js +++ b/get_figma_files_list_core.js @@ -1,5 +1,6 @@ // import json const fs = require('fs') +const fsPromises = require('fs/promises') const path = require('path') const yaml = require('yaml') const date = require('date-and-time') @@ -259,36 +260,35 @@ class FigmaFilesListGetter{ const projects_files = await this.get_projects_files(projects) const files = this.merge_files(projects_files) console.log('files', files) -// const output_file_path = Path(self.config.get( -// 'output_file', './process/figma_files_list.json')) -// const output_limit = self.config.get('output_limit', 0) -// -// if not output_limit or len(files) <= output_limit: -// console.log('Writing the single output file') -// -// with open(output_file_path, 'w', encoding='utf8') as output_file: -// json.dump(files, output_file, ensure_ascii=False, indent=4) -// -// else: -// console.log('Writing multiple output files with partial slices') -// -// parts_count = int((len(files) - 1) / output_limit) + 1 -// -// for part_number in range(1, parts_count + 1): -// start = output_limit * (part_number - 1) -// stop = output_limit * (part_number) -// files_part = files[start:stop] -// -// partial_output_file_path = ( -// output_file_path.parent / -// f'{output_file_path.stem}_part_{part_number}{output_file_path.suffix}' -// ) -// -// with open(partial_output_file_path, 'w', encoding='utf8') as partial_output_file: -// json.dump(files_part, partial_output_file, -// ensure_ascii=False, indent=4) -// -// console.log('Done') + + const output_file_path = this.config.output_file||'./process/figma_files_list.json' + const output_limit = this.config.output_limit||0 + + if(!output_limit || files.length <= output_limit){ + console.log('Writing the single output file') + await fsPromises.writeFile(output_file_path, JSON.stringify(files, null, ' ')) + }else{ + console.log('Writing multiple output files with partial slices') + + const parts_count = (((files.length - 1) / output_limit)|0) + 1 + + for(let part_number = 1; part_number <= parts_count + 1; part_number++){ + const start = output_limit * (part_number - 1) + const stop = output_limit * (part_number) + const files_part = files.slice(start,stop) + + const suffix = path.extname(output_file_path) + const stem = path.basename(output_file_path, suffix) + const parent = path.dirname(output_file_path) + const partial_output_file_path = path.join(parent, `${stem}_part_${part_number}${suffix}`) + + await fsPromises.writeFile(partial_output_file_path, JSON.stringify(files_part, null, ' ')) + // with open(partial_output_file_path, 'w', encoding='utf8') as partial_output_file: + // json.dump(files_part, partial_output_file, + // ensure_ascii=False, indent=4) + } + } + console.log('Done') } } diff --git a/manage.js b/manage.js index fdc570f..f6dedb1 100644 --- a/manage.js +++ b/manage.js @@ -1,6 +1,7 @@ const process = require('process'); const { spawn } = require('child_process') -const fs = require('fs/promises'); +const fs = require('fs'); +const fsPromises = require('fs/promises'); const path = require('path'); const downloadFigmaFiles = require('./download_figma_files_core') const FigmaFilesListGetter = require('./get_figma_files_list_core') @@ -10,13 +11,13 @@ const FigmaFilesListGetter = require('./get_figma_files_list_core') // ======== // Working directory -const working_dir_path= path.resolve('./process1') -const store_dir_path= path.resolve('./store1') +const working_dir_path= path.resolve('./process') +const store_dir_path= path.resolve('./store') const downloads_dir_name = path.normalize(working_dir_path + '/_downloads') const get_list_log_name = path.normalize(working_dir_path + '/get_figma_files_list_log.txt') const download_log_name = path.normalize(store_dir_path + '/download_figma_files_log.txt') -const partial_lists_names = path.normalize(working_dir_path + '/figma_files_list*.json') +// const partial_lists_names = path.normalize(working_dir_path + '/figma_files_list*.json') // Temporary directory const tmp_dir_path = working_dir_path @@ -35,18 +36,16 @@ console.log('args', args) // Other const figma_login = args[0] const figma_password = args[1] -// const current_date = `date +"%Y-%m-%d"` -// -// if (!figma_login){ -// console.error("figma login needed") -// process.exit(1) -// } -// -// if (!figma_password){ -// console.error("figma password needed") -// process.exit(1) -// } +if (!figma_login){ + console.error("figma login needed") + process.exit(1) +} + +if (!figma_password){ + console.error("figma password needed") + process.exit(1) +} // ======================== // Getting files from Figma @@ -57,31 +56,30 @@ const figma_password = args[1] ; (async()=>{ - console.log('downloads_dir_name', working_dir_path) - await fs.rm(working_dir_path, { recursive:true, force:true }) - await fs.mkdir(working_dir_path, { recursive:true }) - await fs.mkdir(store_dir_path, { recursive:true }) + // await fsPromises.rm(working_dir_path, { recursive:true, force:true }) + // await fsPromises.mkdir(working_dir_path, { recursive:true }) + // await fsPromises.mkdir(store_dir_path, { recursive:true }) - const fileListGetter = new FigmaFilesListGetter() - fileListGetter.perform() -})() + // Getting files list + // const fileListGetter = new FigmaFilesListGetter() + // fileListGetter.perform() -// . -// // Getting files list -// ./get_figma_files_list.py 2>&1 | tee ${get_list_log_name} -// -// // Downloading files -// -// list_names=`ls ${partial_lists_names}` -// - -// downloadFigmaFiles() - -// for list_name in ${list_names} ; do -// echo -e "\n\nProcessing the file ${list_name}\n\n" -// node ./download_figma_files.js "figmaLogin=${figma_login}" "figmaPassword=${figma_password}" "figmaFilesList=./${list_name}" "settingFile=./config/download_settings.json" 2>&1 + var files = fs.readdirSync(working_dir_path) + .filter(fn => fn.startsWith('figma_files_list')&&fn.endsWith('.json')) + .map(fn=>path.join(working_dir_path, fn)); + console.log('list files', files) + for(let i in files){ + const list_name = files[i] + console.log(`\n\nProcessing the file ${list_name}\n\n`) + await downloadFigmaFiles({ + figmaFilesListFile:list_name, + figmaLogin:figma_login, + figmaPassword:figma_password, + }) // cp -rf ./${downloads_dir_name}/* ${store_dir_path}/ // sleep 30 -// done + } // -// echo 'DONE' \ No newline at end of file +// echo 'DONE' + +})() diff --git a/package.json b/package.json index 31696ac..b0cb8b5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,9 @@ { "dependencies": { + "date-and-time": "^2.3.0", "fs-extra": "^10.0.1", - "puppeteer": "^13.4.1" + "puppeteer": "^13.4.1", + "xmlhttprequest": "^1.8.0", + "yaml": "^1.10.2" } }