diff --git a/download_figma_files.js b/download_figma_files.js index 79ebeb1..0a7198a 100755 --- a/download_figma_files.js +++ b/download_figma_files.js @@ -1,24 +1,15 @@ #!/usr/bin/node - // Command line arguments - let figmaLogin = ''; let figmaPassword = ''; let figmaFilesListFile = ''; -let downloadsBaseDir = './process/_downloads'; -let debugDir = './process/debug'; +let downloadsBaseDir = ''; +let debugDir = ''; let figmaUrl = ''; let settingFile = ''; -const settings = { - downloadTimeout: 1800, - selectorTimeout: 1500, - navigationTimeout: 180000, - launchTimeout: 120000, - loginTimeout: 10000, - pageOpenTimeout: 10000 -} +const downloadFigmaFiles = require('./download_figma_files_core') process.argv.forEach((arg) => { const keyValue = arg.split("="); @@ -47,328 +38,18 @@ process.argv.forEach((arg) => { } }) - if(!figmaLogin || !figmaPassword || (!figmaFilesListFile && !figmaUrl)) { console.log('Usage: []'); process.exit(1); } -const fs = require('fs'); -const fse = require('fs-extra') - -// Creating the directory for debug purposes - -if(debugDir) { - if(!debugDir.endsWith('/')) { - debugDir += '/'; - } - fs.mkdirSync(debugDir, {recursive: true}); - - console.log('Working in debug mode; content and screenshots of the file pages will be saved to the directory: ' + debugDir + '\n'); -} - - -// Reading Figma files list file - -let figmaFilesList; - -if (figmaFilesListFile) { - fs.readFile(figmaFilesListFile, (err, data) => { - if(err) throw err; - figmaFilesList = JSON.parse(data); - }); -} -if (figmaUrl) { - figmaFilesList = [ - { - uri: figmaUrl - } - ] -} - -if (settingFile) { - let newSettings = {} - const data = fs.readFileSync(settingFile); - newSettings = JSON.parse(data); - - settings.downloadTimeout = newSettings.downloadTimeout ?? settings.downloadTimeout; - settings.selectorTimeout = newSettings.selectorTimeout ?? settings.selectorTimeout; - settings.navigationTimeout = newSettings.navigationTimeout ?? settings.navigationTimeout; - settings.launchTimeout = newSettings.launchTimeout ?? settings.launchTimeout; - settings.loginTimeout = newSettings.loginTimeout ?? settings.loginTimeout; - settings.pageOpenTimeout = newSettings.pageOpenTimeout ?? settings.pageOpenTimeout; -} - -console.log('settings: ', settings); - -// Login page settings -const loginPageUrl = 'https://www.figma.com/login'; -const usernameSelector = 'input[name="email"]'; -const passwordSelector = 'input[name="password"]'; -const buttonSelector = 'button[type="submit"]'; - - -// After-login page settings -const filebrowserSelector = '#filebrowser-loading-page'; - - -// File page settings -const filePageBaseUrl = 'https://www.figma.com/file/'; -const fullscreenFilenameSelector = '#fullscreen-filename'; - - -// Using Puppeteer - -const puppeteer = require('puppeteer'); - -(async() => { - // Launching Chrome - const browser = await puppeteer.launch({args: ['--no-sandbox', '--disable-setuid-sandbox'], timeout: settings.launchTimeout}); - const page = await browser.newPage(); - page.setDefaultNavigationTimeout(settings.navigationTimeout); - - try { - // Opening the login page - console.log('Opening the login page'); - await page.goto(loginPageUrl, {waitUntil: 'networkidle2'}); - - - // Filling the email field - console.log('Filling the email field'); - await page.focus(usernameSelector, {delay: 500}); - await page.keyboard.type(figmaLogin); - - - // Filling the password field - console.log('Filling the password field'); - await page.focus(passwordSelector, {delay: 500}); - await page.keyboard.type(figmaPassword); - - - // Clicking the submit button - console.log('Clicking the submit button'); - await page.click(buttonSelector, {delay: 500}); - console.log('Waiting for an after-login page opening'); - - - // Sleeping for 10 seconds - await sleep(settings.loginTimeout); - - - // Waiting for an after-login page loading - await page.waitForSelector(filebrowserSelector); - } catch (err) { - console.error(err); - process.exit(1); - } - - if (!downloadsBaseDir.endsWith('/')) { - downloadsBaseDir += '/' - } - - let tmpDownloadDir = `${downloadsBaseDir}tmp/`; - - for (let t = 1; t < 1000; t++) { - if (!fs.existsSync(tmpDownloadDir)) { - break; - } - tmpDownloadDir = `${downloadsBaseDir}tmp${t}/`; - } - fs.mkdirSync(tmpDownloadDir, {recursive: true}); - - console.log(`Tmp directory to save the file: ${tmpDownloadDir}`); - - // Processing Figma files - - for(let i = 0; i < figmaFilesList.length; i++) { - let filePage = "" - if (figmaFilesList[i].uri && figmaFilesList[i].uri.startsWith("http")) { - filePage = figmaFilesList[i].uri - } else if (figmaFilesList[i].uri) { - filePage = filePageBaseUrl + figmaFilesList[i].uri - if (!filePage.endsWith("/")) { - filePage += "/" - } - } else { - filePage = filePageBaseUrl + figmaFilesList[i].key + '/'; - } - - console.log('\nStarting to process new file (' + (i + 1) + '/' + figmaFilesList.length + '), url: ' + filePage); - - try { - // Opening a file page - console.log('Opening the file page: ' + filePage); - let filePageResponse = await page.goto(filePage, {waitUntil: 'networkidle2'}); - - - // Checking status code - console.log('Returned status: ' + filePageResponse.status()); - if(filePageResponse.status() !== 200) { - console.log('Skipping the file'); - continue; - } - // Sleeping for 10 seconds, waiting for a specific element in React-generated content - await sleep(settings.pageOpenTimeout); - await page.waitForSelector(fullscreenFilenameSelector); - - // Checking if the file is available to save locally - - let content = await page.content(); - - if(content.includes('="Viewers can\'t copy or share this file."')) { - console.log('This file is protected against saving locally and sharing. Skipping') - continue; - } - // Getting and validating page title - - const title = await page.title(); - if(!title.endsWith(' – Figma')) { - console.log(`Title format "${title}" seems to be unrecognized, skipping the file`) - await page.screenshot({path: debugDir + 'login_screenshot' + '.png', fullPage: true}); - continue; - } - const fileName = title.replace(' – Figma', '').replaceAll('/', '_').replaceAll('|', '_').replaceAll('"', '_'); - // Getting the local path of the directory for the file to download - - let downloadDir = downloadsBaseDir; - - if(!downloadDir.endsWith('/')) { - downloadDir += '/'; - } - - if(figmaFilesList[i].team) { - downloadDir += 'TEAM ' + figmaFilesList[i].team + '/'; - } - - if(figmaFilesList[i].project) { - downloadDir += 'PROJECT ' + figmaFilesList[i].project + '/'; - } - if (!figmaFilesList[i].team && !figmaFilesList[i].project) { - downloadDir += fileName + '/'; - } - - - - console.log(`Directory to save the file: ${downloadDir}`); - - fs.mkdirSync(downloadDir, {recursive: true}); - - // Set download behavior - await page._client.send('Page.setDownloadBehavior', {behavior: 'allow', downloadPath: tmpDownloadDir}); - - // Debug: making screenshot and saving the page content - if(debugDir) { - fs.writeFile(debugDir + (i + 1) + '_content' + '.html', content, () => {}); - await page.screenshot({path: debugDir + (i + 1) + '_screenshot' + '.png', fullPage: true}); - } - - - await page.evaluate(_ => { - const mainMenu = document.querySelector('div[data-tooltip="main-menu"]'); - const clickEvt = document.createEvent("MouseEvents"); - clickEvt.initEvent("mousedown", true, true); - mainMenu.dispatchEvent(clickEvt); - }); - - await sleep(200); - - let menuItemFileHandle = null; - try { - menuItemFileHandle = await page.waitForSelector('div[data-testid="dropdown-option-File"]', {timeout: settings.selectorTimeout}); - } catch (error) { - if(debugDir && !menuItemFileHandle) { - console.log('cannot open main menu') - fs.writeFile(debugDir + fileName + '_main_menu' + '.html', content, () => {}); - await page.screenshot({path: debugDir + fileName + '_main_menu_screenshot' + '.png', fullPage: true}); - } - throw error; - } - - const box = await menuItemFileHandle.boundingBox(); - await page.mouse.move(box.x + 5, box.y + 5); - - await sleep(200); - let submenuFileHandle = null - try { - submenuFileHandle = await page.waitForSelector('div[data-testid="dropdown-option-Save local copy…"]', {timeout: settings.selectorTimeout}); - } catch (error) { - try { - const newFileItem = await page.waitForSelector('div[data-testid="dropdown-option-New design file"]', {timeout: settings.selectorTimeout}); - if (newFileItem) { - console.log('cannot save copy export not allowed') - continue; - } - } catch (error) { - //ignore - } - - if(debugDir && !submenuFileHandle) { - console.log('cannot open file menu') - fs.writeFile(debugDir + fileName + '_file_menu' + '.html', content, () => {}); - await page.screenshot({path: debugDir + fileName + '_file_menu_screenshot' + '.png', fullPage: true}); - } - throw error; - } - - - const saveFileBox = await submenuFileHandle.boundingBox(); - await page.mouse.move(saveFileBox.x + 5, saveFileBox.y + 5); - await page.mouse.click(saveFileBox.x + 5, saveFileBox.y + 5); - - let downloadedCheckTries = settings.downloadTimeout; - - let downloaded = false; - for(let j = 0; downloadedCheckTries == 0 || j < downloadedCheckTries; j++) { - - await sleep(1000); - fs.readdir(tmpDownloadDir, (err, files) => { - let donwloadedFile = '' - if (files && files.length > 0) { - donwloadedFile = files[0] - } - - if (donwloadedFile.length) { - if (fs.existsSync(downloadDir + donwloadedFile)) { - const tmpFileName = donwloadedFile.lastIndexOf('.'); - const now = new Date().toISOString().substring(0, 19).replaceAll('T', '_').replaceAll(':','-'); - fse.moveSync(downloadDir + donwloadedFile, downloadDir + donwloadedFile.substring(0, tmpFileName) + '_' + now + donwloadedFile.substring(tmpFileName)) - } - fse.moveSync(tmpDownloadDir + donwloadedFile, downloadDir + donwloadedFile); - - downloaded = true; - } - }); - - if (downloaded) { - console.log('Download complete'); - break; - } - - if (j % 30 == 0) { - console.log(`waiting file to download for ${parseInt(j / 60)} min ${j % 60} sec.`) - } - - if(downloadedCheckTries > 0 && j === (downloadedCheckTries - 1)) { - console.log(`File ${title} is not downloaded during timeout`) - } - } - } catch (err) { - console.error(err); - continue; - } - } - fs.rmdirSync(tmpDownloadDir); - - // Closing Chromium - browser.close(); -})(); - - -// Sleep function - -function sleep(ms) { - return new Promise(resolve => { - setTimeout(resolve, ms); - }) -} +downloadFigmaFiles({ + figmaLogin, + figmaPassword, + figmaFilesListFile, + figmaUrl, + downloadsBaseDir, + debugDir, + settingFile, +}) \ No newline at end of file diff --git a/download_figma_files_core.js b/download_figma_files_core.js new file mode 100644 index 0000000..8fc948a --- /dev/null +++ b/download_figma_files_core.js @@ -0,0 +1,332 @@ + +const path = require('path'); +const fs = require('fs'); +const fse = require('fs-extra') +const puppeteer = require('puppeteer'); + +function sleep(ms) { + return new Promise(resolve => { + setTimeout(resolve, ms); + }) +} + +const settings = { + downloadTimeout: 1800, + selectorTimeout: 20000, + navigationTimeout: 180000, + launchTimeout: 120000, + loginTimeout: 10000, + pageOpenTimeout: 20000 +} + +const downloadFigmaFiles = async(config = {}) => { + + let { + figmaLogin, + figmaPassword, + figmaFilesListFile, + figmaUrl, + downloadsBaseDir = './process/_downloads', + debugDir = './process/debug', + settingFile, + } = config + + console.log('figmaFilesListFile', figmaFilesListFile) + + + if(!figmaLogin || !figmaPassword || (!figmaFilesListFile && !figmaUrl)) + { + console.log('Usage: []'); + process.exit(1); + } + + // Creating the directory for debug purposes + + if(debugDir) { + if(!debugDir.endsWith('/')) { + debugDir += '/'; + } + fs.mkdirSync(debugDir, {recursive: true}); + + console.log('Working in debug mode; content and screenshots of the file pages will be saved to the directory: ' + debugDir + '\n'); + } + + + // Reading Figma files list file + + if (figmaFilesListFile) { + fs.readFile(figmaFilesListFile, (err, data) => { + if(err) throw err; + figmaFilesList = JSON.parse(data); + }); + } + if (figmaUrl) { + figmaFilesList = [ + { + uri: figmaUrl + } + ] + } + + if (settingFile) { + let newSettings = {} + const data = fs.readFileSync(settingFile); + newSettings = JSON.parse(data); + + settings.downloadTimeout = newSettings.downloadTimeout ?? settings.downloadTimeout; + settings.selectorTimeout = newSettings.selectorTimeout ?? settings.selectorTimeout; + settings.navigationTimeout = newSettings.navigationTimeout ?? settings.navigationTimeout; + settings.launchTimeout = newSettings.launchTimeout ?? settings.launchTimeout; + settings.loginTimeout = newSettings.loginTimeout ?? settings.loginTimeout; + settings.pageOpenTimeout = newSettings.pageOpenTimeout ?? settings.pageOpenTimeout; + } + + console.log('settings: ', settings); + + // Login page settings + const loginPageUrl = 'https://www.figma.com/login'; + const usernameSelector = 'input[name="email"]'; + const passwordSelector = 'input[name="password"]'; + const buttonSelector = 'button[type="submit"]'; + + + // After-login page settings + const filebrowserSelector = '#filebrowser-loading-page'; + + + // File page settings + const filePageBaseUrl = 'https://www.figma.com/file/'; + const fullscreenFilenameSelector = '#fullscreen-filename'; + + + // Launching Chrome + const browser = await puppeteer.launch({args: ['--no-sandbox', '--disable-setuid-sandbox'], timeout: settings.launchTimeout}); + const page = await browser.newPage(); + page.setDefaultNavigationTimeout(settings.navigationTimeout); + + try { + // Opening the login page + console.log('Opening the login page'); + await page.goto(loginPageUrl, {waitUntil: 'networkidle2'}); + + + // Filling the email field + console.log('Filling the email field'); + await page.focus(usernameSelector, {delay: 500}); + await page.keyboard.type(figmaLogin); + + + // Filling the password field + console.log('Filling the password field'); + await page.focus(passwordSelector, {delay: 500}); + await page.keyboard.type(figmaPassword); + + await page.screenshot({path: debugDir + 'login_form_screenshot' + '.png', fullPage: true}); + + // Clicking the submit button + console.log('Clicking the submit button'); + await page.click(buttonSelector, {delay: 500}); + console.log('Waiting for an after-login page opening'); + + + // Sleeping for 10 seconds + await sleep(settings.loginTimeout); + + await page.screenshot({path: debugDir + 'login_form_after_screenshot' + '.png', fullPage: true}); + + // Waiting for an after-login page loading + await page.waitForSelector(filebrowserSelector); + } catch (err) { + console.error(err); + process.exit(1); + } + + // Processing Figma files + + for(let i = 0; i < figmaFilesList.length; i++) { + let filePage = "" + if (figmaFilesList[i].uri && figmaFilesList[i].uri.startsWith("http")) { + filePage = figmaFilesList[i].uri + } else if (figmaFilesList[i].uri) { + filePage = filePageBaseUrl + figmaFilesList[i].uri + if (!filePage.endsWith("/")) { + filePage += "/" + } + } else { + filePage = filePageBaseUrl + figmaFilesList[i].key + '/'; + } + + console.log('\nStarting to process new file (' + (i + 1) + '/' + figmaFilesList.length + '), url: ' + filePage); + + try { + // Opening a file page + console.log('Opening the file page: ' + filePage); + let filePageResponse = await page.goto(filePage, {waitUntil: 'networkidle2'}); + + + // Checking status code + console.log('Returned status: ' + filePageResponse.status()); + if(filePageResponse.status() !== 200) { + console.log('Skipping the file'); + continue; + } + // Sleeping for 10 seconds, waiting for a specific element in React-generated content + await sleep(settings.pageOpenTimeout); + await page.waitForSelector(fullscreenFilenameSelector); + + // Checking if the file is available to save locally + + let content = await page.content(); + + if(content.includes('="Viewers can\'t copy or share this file."')) { + console.log('This file is protected against saving locally and sharing. Skipping') + continue; + } + // Getting and validating page title + + const title = await page.title(); + if(!title.endsWith(' – Figma')) { + console.log(`Title format "${title}" seems to be unrecognized, skipping the file`) + await page.screenshot({path: debugDir + 'login_screenshot' + '.png', fullPage: true}); + continue; + } + const fileName = title.replace(' – Figma', '').replaceAll('/', '_').replaceAll('|', '_').replaceAll('"', '_'); + // Getting the local path of the directory for the file to download + + let downloadDir = path.resolve(downloadsBaseDir); + + if(!downloadDir.endsWith('/')) { + downloadDir += '/'; + } + + if(figmaFilesList[i].team) { + downloadDir += 'TEAM ' + figmaFilesList[i].team + '/'; + } + + if(figmaFilesList[i].project) { + downloadDir += 'PROJECT ' + figmaFilesList[i].project + '/'; + } + if (!figmaFilesList[i].team && !figmaFilesList[i].project) { + downloadDir += fileName + '/'; + } + + downloadDir = path.normalize(downloadDir) + + + if (fs.existsSync(downloadDir + fileName+'.fig')) { + const now = new Date().toISOString().substring(0, 19).replaceAll('T', '_').replaceAll(':','-'); + fse.moveSync(downloadDir + fileName+'.fig', downloadDir + fileName + '_' + now + '.fig') + } + + if (fs.existsSync(downloadDir + fileName+'.jam')) { + const now = new Date().toISOString().substring(0, 19).replaceAll('T', '_').replaceAll(':','-'); + fse.moveSync(downloadDir + fileName+'.jam', downloadDir + fileName + '_' + now + '.jam') + } + + + console.log(`Directory to save the file: ${downloadDir}${fileName}`); + + fs.mkdirSync(downloadDir, {recursive: true}); + console.log(`created`); + + + // Set download behavior + await page._client.send('Page.setDownloadBehavior', {behavior: 'allow', downloadPath: downloadDir}); + console.log(`Page.setDownloadBehavior done`); + + // Debug: making screenshot and saving the page content + if(debugDir) { + fs.writeFile(debugDir + (i + 1) + '_content' + '.html', content, () => {}); + await page.screenshot({path: debugDir + fileName + (i + 1) + '_screenshot' + '.png', fullPage: true}); + } + + await page.evaluate(_ => { + const mainMenu = document.querySelector('div[data-tooltip="main-menu"]'); + const clickEvt = document.createEvent("MouseEvents"); + clickEvt.initEvent("mousedown", true, true); + mainMenu.dispatchEvent(clickEvt); + }); + + await sleep(1000); + + + console.log(`looking for download button`); + + let menuItemFileHandle = null; + try { + menuItemFileHandle = await page.waitForSelector('div[data-testid="dropdown-option-File"]', {timeout: settings.selectorTimeout}); + } catch (error) { + if(debugDir && !menuItemFileHandle) { + console.log('cannot open main menu') + fs.writeFile(debugDir + fileName + '_main_menu' + '.html', content, () => {}); + await page.screenshot({path: debugDir + fileName + '_main_menu_screenshot' + '.png', fullPage: true}); + } + throw error; + } + + const box = await menuItemFileHandle.boundingBox(); + await page.mouse.move(box.x + 5, box.y + 5); + + await sleep(200); + let submenuFileHandle = null + try { + submenuFileHandle = await page.waitForSelector('div[data-testid="dropdown-option-Save local copy…"]', {timeout: settings.selectorTimeout}); + } catch (error) { + try { + const newFileItem = await page.waitForSelector('div[data-testid="dropdown-option-New design file"]', {timeout: settings.selectorTimeout}); + if (newFileItem) { + console.log('cannot save copy export not allowed') + continue; + } + } catch (error) { + //ignore + } + + if(debugDir && !submenuFileHandle) { + console.log('cannot open file menu') + fs.writeFile(debugDir + fileName + '_file_menu' + '.html', content, () => {}); + await page.screenshot({path: debugDir + fileName + '_file_menu_screenshot' + '.png', fullPage: true}); + } + throw error; + } + + const saveFileBox = await submenuFileHandle.boundingBox(); + await page.mouse.move(saveFileBox.x + 5, saveFileBox.y + 5); + await page.mouse.click(saveFileBox.x + 5, saveFileBox.y + 5); + + await page.screenshot({path: debugDir + fileName + '_after_savefile' + '.png', fullPage: true}); + + let downloadedCheckTries = settings.downloadTimeout; + + for(let j = 0; downloadedCheckTries == 0 || j < downloadedCheckTries; j++) { + await page.screenshot({path: debugDir + fileName + '_dl_try_' + j + '.png', fullPage: true}); + + await sleep(1000); + if (fs.existsSync(downloadDir + fileName+'.fig')) { + console.log('Download complete'); + break; + } + if (fs.existsSync(downloadDir + fileName+'.jam')) { + console.log('Download complete'); + break; + } + + if (j % 30 == 0) { + console.log(`waiting file to download for ${parseInt(j / 60)} min ${j % 60} sec.`) + } + + if(downloadedCheckTries > 0 && j === (downloadedCheckTries - 1)) { + console.log(`File ${downloadDir + fileName} is not downloaded during timeout`) + } + } + } catch (err) { + console.error(err); + continue; + } + } + + // Closing Chromium + browser.close(); +} + +module.exports = downloadFigmaFiles diff --git a/get_figma_files_list.js b/get_figma_files_list.js new file mode 100644 index 0000000..fe60741 --- /dev/null +++ b/get_figma_files_list.js @@ -0,0 +1,3 @@ +const FigmaFilesListGetter = require('./get_figma_files_list_core') +const fileListGetter = new FigmaFilesListGetter() +fileListGetter.perform() diff --git a/get_figma_files_list_core.js b/get_figma_files_list_core.js new file mode 100644 index 0000000..b28eb90 --- /dev/null +++ b/get_figma_files_list_core.js @@ -0,0 +1,298 @@ +// import json +const fs = require('fs') +const fsPromises = require('fs/promises') +const path = require('path') +const yaml = require('yaml') +const date = require('date-and-time') +const { XMLHttpRequest } = require('xmlhttprequest') + +// import shutil +// from datetime import datetime +// from pathlib import Path +// from urllib import request +// from urllib.error import HTTPError +// from yaml import load, Loader + +const fetch = (url, opts={})=>{ + console.log('fetch', url) + const { mode='GET', headers } = opts + return new Promise((resolve, reject)=>{ + var xhttp = new XMLHttpRequest(); + + xhttp.onreadystatechange = function() { + // console.log('onreadystatechange', this.readyState, this.status) + if (this.readyState == 4 && this.status == 200) { + // Typical action to be performed when the document is ready: + // document.getElementById("demo").innerHTML = xhttp.responseText; + // console.log('respo', xhttp.responseText) + resolve(xhttp.responseText) + } + }; + xhttp.open(mode, url, true); + headers.forEach(([k, v])=>{ + // console.log('kv', k, v) + xhttp.setRequestHeader(k, v) + }) + xhttp.send(); + }) +} + +class FigmaFilesListGetter{ + constructor(config_file_path = path.resolve('./config/get_figma_files_list.yml')){ + console.log('FigmaFilesListGetter cfg', config_file_path) + const config_file = fs.readFileSync(config_file_path, 'utf-8') + console.log('config_file', config_file) + this.config = yaml.parse(config_file) + + this.token_headers = [['X-Figma-Token', this.config['access_token']||'']] + } + + async get_team_projects(team_id){ + try{ + + const api_request_url = `https://api.figma.com/v1/teams/${team_id}/projects` + + console.log(`Getting team projects, requesting URL: ${api_request_url}`) + + const api_response = await fetch( + api_request_url, + { + mode:'GET', + headers: this.token_headers + }) + + +// console.log('api_response', api_response) +// + const api_response_data = JSON.parse(api_response) + console.log(api_response_data) + + const team_name = api_response_data.name||team_id + const team_projects = [] + + const { projects=[] } = api_response_data + + for(let i in projects){ + team_projects.push( + { + 'id': projects[i].id, + 'team': team_name + } + ) + } + + return team_projects + + } catch(err){ + console.error(err) + return '' + } + } + + async get_teams_projects(){ + console.log('Getting the projects of all teams listed in config, if any, removing duplicates') + + let teams_projects = [] + + const teams = this.config['teams']||[] + console.log('teams', teams) + for(let i in teams){ + const team_projects = await this.get_team_projects(teams[i]) + teams_projects = teams_projects.concat(team_projects) + } + + return teams_projects + } + + merge_projects(teams_projects){ + console.log('Merging teams projects with projects listed in config, if any, removing duplicates') + + const merged_projects = [] + const seen_projects_ids = [] + + for(let i in teams_projects){ + const project = teams_projects[i] + if(!seen_projects_ids.find(id=>id == project.id)){ + merged_projects.push(project) + seen_projects_ids.push(project.id) + } + } +// +// for(let project_id in list(set(self.config.get('projects', [])))){ +// project_id = str(project_id) +// +// if(project_id not in seen_projects_ids){ +// merged_projects.append( +// { +// 'id': project_id, +// 'team': '' +// } +// ) +// +// seen_projects_ids.add(project_id) +// } +// } + + return merged_projects + } + + async get_project_files(project){ + const api_request_url = `https://api.figma.com/v1/projects/${project["id"]}/files` + const today = date.format(new Date(), 'YY-MM-DD') + + console.log(`Getting project files, requesting URL: ${api_request_url}`, today) + + const api_response = await fetch( + api_request_url, + { + mode:'GET', + headers: this.token_headers + }) + + const api_response_data = JSON.parse(api_response) + + // console.log('api_response_data', api_response_data) + + // if(api_response['status'] != 200 or api_response_data.get('err', None)){ + // console.log('Failed to perform API request') + // return [] + // } + + const project_name = api_response_data.name||project.id + const project_files = [] + + const { files } = api_response_data + + for(let i in files){ + const file = files[i] + + const file_name_to_check = `./store/TEAM ${project["team"]}/PROJECT ${project_name}/${file["name"]}.fig` + // let time = 0 + // if(os.path.isfile(file_name_to_check)){ + // time = os.path.getmtime(file_name_to_check) + // } + // const updates = datetime.strptime( + // file["last_modified"], "%Y-%m-%dT%H:%M:%SZ").timestamp() + // console.log(f'time: {time}, updated: {updates}') + // if(time < updates){ + // console.log(`File TEAM ${project["team"]}/PROJECT ${project_name}/${file["name"]}, key ${file["key"]}, last modified: ${file["last_modified"]} -> adding to the list`) + project_files.push( + { + 'key': file.key, + 'name': file.name, + 'project': project_name, + 'team': project.team, + 'last_modified': file['last_modified'] + } + ) + // if(os.path.isfile(file_name_to_check)){ + // if not os.path.exists(f'./store/{today}'): + // os.makedirs(f'./store/{today}') + // if not os.path.exists(f'./store/{today}/TEAM {project["team"]}'): + // os.makedirs(f'./store/{today}/TEAM {project["team"]}') + // if not os.path.exists(f'./store/{today}/TEAM {project["team"]}/PROJECT {project_name}/'): + // os.makedirs( + // f'./store/{today}/TEAM {project["team"]}/PROJECT {project_name}/') + // shutil.copyfile( + // file_name_to_check, f'./store/{today}/TEAM {project["team"]}/PROJECT {project_name}/{file["name"]}.fig') + // os.remove(file_name_to_check) + // } + // }else{ + // console.log(`File ${project["team"]}/${project_name}/${file["name"]}, key ${file["key"]}, last modified: ${file["last_modified"]} not modifided`) + // } + } + + return project_files + } + + async get_projects_files(projects){ + console.log('Getting the files of all projects') + + let projects_files = [] + + for( let i in projects ){ + const project_files = await this.get_project_files(projects[i]) + projects_files = projects_files.concat(project_files) + } + + return projects_files + } + + merge_files(projects_files){ + console.log('Merging projects files with files listed in config, if any, removing duplicates') + + const merged_files = [] + const seen_files_keys = [] + + for(let i in projects_files){ + const file = projects_files[i] + if(!seen_files_keys.find(v=>v==file.key)){ + merged_files.push(file) + seen_files_keys.push(file.key) + } + } + + const file_keys = this.config.files||[] + for(let i in file_keys){ + const file_key = ""+file_keys[i] + + if(!seen_files_keys.find(v=>v==file_key)){ + merged_files.push( + { + 'key': file_key, + 'project': '', + 'team': '' + } + ) + + seen_files_keys.push(file_key) + } + } + + return merged_files + } + + async perform(){ + const teams_projects = await this.get_teams_projects() + // console.log('teams_projects', teams_projects) + const projects = this.merge_projects(teams_projects) + // console.log('projects', projects) + const projects_files = await this.get_projects_files(projects) + const files = this.merge_files(projects_files) + console.log('files', files) + + const output_file_path = this.config.output_file||'./process/figma_files_list.json' + const output_limit = this.config.output_limit||0 + + if(!output_limit || files.length <= output_limit){ + console.log('Writing the single output file') + await fsPromises.writeFile(output_file_path, JSON.stringify(files, null, ' ')) + }else{ + console.log('Writing multiple output files with partial slices') + + const parts_count = (((files.length - 1) / output_limit)|0) + 1 + + for(let part_number = 1; part_number <= parts_count + 1; part_number++){ + const start = output_limit * (part_number - 1) + const stop = output_limit * (part_number) + const files_part = files.slice(start,stop) + + const suffix = path.extname(output_file_path) + const stem = path.basename(output_file_path, suffix) + const parent = path.dirname(output_file_path) + const partial_output_file_path = path.join(parent, `${stem}_part_${part_number}${suffix}`) + + await fsPromises.writeFile(partial_output_file_path, JSON.stringify(files_part, null, ' ')) + // with open(partial_output_file_path, 'w', encoding='utf8') as partial_output_file: + // json.dump(files_part, partial_output_file, + // ensure_ascii=False, indent=4) + } + } + console.log('Done') + + } +} +// if __name__ == "__main__": +// FigmaFilesListGetter().perform() + +module.exports = FigmaFilesListGetter \ No newline at end of file diff --git a/manage.js b/manage.js new file mode 100644 index 0000000..f6dedb1 --- /dev/null +++ b/manage.js @@ -0,0 +1,85 @@ +const process = require('process'); +const { spawn } = require('child_process') +const fs = require('fs'); +const fsPromises = require('fs/promises'); +const path = require('path'); +const downloadFigmaFiles = require('./download_figma_files_core') +const FigmaFilesListGetter = require('./get_figma_files_list_core') + +// ======== +// Settings +// ======== + +// Working directory +const working_dir_path= path.resolve('./process') +const store_dir_path= path.resolve('./store') + +const downloads_dir_name = path.normalize(working_dir_path + '/_downloads') +const get_list_log_name = path.normalize(working_dir_path + '/get_figma_files_list_log.txt') +const download_log_name = path.normalize(store_dir_path + '/download_figma_files_log.txt') +// const partial_lists_names = path.normalize(working_dir_path + '/figma_files_list*.json') + +// Temporary directory +const tmp_dir_path = working_dir_path + +// HTTP server directory +const http_server_dir_path = './figma' +const deduplicated_dir_name = 'deduplicated' +const stat_log_name = 'stats_log.txt' +const archive_dirs_to_keep = 30 +const archive_dir_name_pattern = '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]' +const df_line_to_grep = '/dev/mapper/centos-root' + +const args = process.argv.slice(2) +console.log('args', args) + +// Other +const figma_login = args[0] +const figma_password = args[1] + +if (!figma_login){ + console.error("figma login needed") + process.exit(1) +} + +if (!figma_password){ + console.error("figma password needed") + process.exit(1) +} + +// ======================== +// Getting files from Figma +// ======================== + +// Coming to the working directory +// Remove files that may be kept since previous session +; +(async()=>{ + + // await fsPromises.rm(working_dir_path, { recursive:true, force:true }) + // await fsPromises.mkdir(working_dir_path, { recursive:true }) + // await fsPromises.mkdir(store_dir_path, { recursive:true }) + + // Getting files list + // const fileListGetter = new FigmaFilesListGetter() + // fileListGetter.perform() + + var files = fs.readdirSync(working_dir_path) + .filter(fn => fn.startsWith('figma_files_list')&&fn.endsWith('.json')) + .map(fn=>path.join(working_dir_path, fn)); + console.log('list files', files) + for(let i in files){ + const list_name = files[i] + console.log(`\n\nProcessing the file ${list_name}\n\n`) + await downloadFigmaFiles({ + figmaFilesListFile:list_name, + figmaLogin:figma_login, + figmaPassword:figma_password, + }) +// cp -rf ./${downloads_dir_name}/* ${store_dir_path}/ +// sleep 30 + } +// +// echo 'DONE' + +})() diff --git a/package.json b/package.json index 31696ac..b0cb8b5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,9 @@ { "dependencies": { + "date-and-time": "^2.3.0", "fs-extra": "^10.0.1", - "puppeteer": "^13.4.1" + "puppeteer": "^13.4.1", + "xmlhttprequest": "^1.8.0", + "yaml": "^1.10.2" } }