From ee5316e6efcb99ba8c001845879d826e30524866 Mon Sep 17 00:00:00 2001 From: Wyatt Johnson Date: Tue, 27 Mar 2018 18:28:14 -0600 Subject: [PATCH] initial download support --- graph/resolvers/comment.js | 7 +++ graph/typeDefs.graphql | 3 + package.json | 2 + routes/api/v1/account.js | 113 +++++++++++++++++++++++++++++++++++++ yarn.lock | 84 +++++++++++++++++++++++++-- 5 files changed, 204 insertions(+), 5 deletions(-) diff --git a/graph/resolvers/comment.js b/graph/resolvers/comment.js index 7005701ec..2e4d9a08d 100644 --- a/graph/resolvers/comment.js +++ b/graph/resolvers/comment.js @@ -1,3 +1,4 @@ +const { URL } = require('url'); const { property } = require('lodash'); const { SEARCH_ACTIONS } = require('../../perms/constants'); const { decorateWithTags, decorateWithPermissionCheck } = require('./util'); @@ -55,6 +56,12 @@ const Comment = { editableUntil: editableUntil, }; }, + async url(comment, args, { loaders: { Assets } }) { + const asset = await Assets.getByID.load(comment.asset_id); + const assetURL = new URL(asset.url); + assetURL.searchParams.set('commentId', comment.id); + return assetURL.href; + }, }; // Decorate the Comment type resolver with a tags field. diff --git a/graph/typeDefs.graphql b/graph/typeDefs.graphql index 2414faf1f..588ad08dc 100644 --- a/graph/typeDefs.graphql +++ b/graph/typeDefs.graphql @@ -547,6 +547,9 @@ type Comment { # Indicates if it has a parent hasParent: Boolean + + # url is the permalink to this particular Comment on the Asset. + url: String } # CommentConnection represents a paginable subset of a comment list. diff --git a/package.json b/package.json index 0253c7b0b..4436ec921 100644 --- a/package.json +++ b/package.json @@ -62,6 +62,7 @@ "apollo-server-express": "^1.2.0", "apollo-utilities": "^1.0.3", "app-module-path": "^2.2.0", + "archiver": "^2.1.1", "autoprefixer": "^6.5.2", "babel-cli": "6.26.0", "babel-core": "6.26.0", @@ -92,6 +93,7 @@ "copy-webpack-plugin": "^4.0.0", "cross-spawn": "^5.1.0", "css-loader": "^0.28.5", + "csv-stringify": "^2.0.4", "dataloader": "^1.3.0", "debug": "3.1.0", "dialog-polyfill": "^0.4.9", diff --git a/routes/api/v1/account.js b/routes/api/v1/account.js index 561134885..84b5e4d0a 100644 --- a/routes/api/v1/account.js +++ b/routes/api/v1/account.js @@ -127,4 +127,117 @@ router.put( } ); +const archiver = require('archiver'); +const stringify = require('csv-stringify'); +const moment = require('moment'); +const { pick, get, kebabCase } = require('lodash'); + +// loadCommentsBatch will load a batch of the comments and write them to the +// stream. +async function loadCommentsBatch(ctx, csv, variables = {}) { + let result = await ctx.graphql( + ` + query GetMyComments($cursor: Cursor) { + me { + comments(query: { + limit: 100, + cursor: $cursor + }) { + hasNextPage + endCursor + nodes { + id + created_at + asset { + url + } + body + url + } + } + } + } + `, + variables + ); + if (result.errors) { + throw result.errors; + } + + for (const comment of get(result, 'data.me.comments.nodes', [])) { + csv.write([ + comment.id, + moment(comment.created_at).format('YYYY-MM-DD HH:mm:ss'), + comment.asset.url, + comment.url, + comment.body, + ]); + } + + return pick(result.data.me.comments, ['hasNextPage', 'endCursor']); +} + +// loadComments will load batches of the comments and write them to the csv +// stream. Once the comments have finished writing, it will close the stream. +async function loadComments(ctx, csv) { + csv.write(['ID', 'Timestamp', 'Article', 'Link', 'Body']); + + // Load the first batch's comments. + let connection = await loadCommentsBatch(ctx, csv); + + // As long as there's more comments, keep paginating. + while (connection.hasNextPage) { + connection = await loadCommentsBatch(ctx, csv, { + cursor: connection.endCursor, + }); + } + + csv.end(); +} + +// /download will send back a zipped archive of the users account. +router.get('/download', authorization.needed(), async (req, res, next) => { + try { + const result = await req.context.graphql('{ me { username } }'); + if (result.errors) { + throw result.errors; + } + const username = get(result, 'data.me.username'); + + // Generate the filename of the file that the user will download. + const filename = `talk-${kebabCase(username)}-${kebabCase( + moment().format('YYYY-MM-DD HH:mm:ss') + )}.zip`; + + res.writeHead(200, { + 'Content-Type': 'application/octet-stream', + 'Content-Disposition': `attachment; filename=${filename}`, + }); + + // Create the zip archive we'll use to write all the exported files to. + const archive = archiver('zip', { + zlib: { level: 9 }, + }); + + // Pipe this to the response writer directly. + archive.pipe(res); + + // Create all the csv writers that'll write the data to the archive. + const myCommentsCSV = stringify(); + + // Add all the streams as files to the archive. + archive.append(myCommentsCSV, { name: 'my_comments.csv' }); + + // Mark the end of adding files, no more files can be added after this. Once + // all the stream readers have finished writing, and have closed, the + // archiver will close which will finish the HTTP request. + archive.finalize(); + + // Load the comments csv up with the user's comments. + await loadComments(req.context, myCommentsCSV); + } catch (err) { + return next(err); + } +}); + module.exports = router; diff --git a/yarn.lock b/yarn.lock index fd501351e..199e329f8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -445,6 +445,30 @@ aproba@^1.0.3, aproba@^1.1.1: version "1.2.0" resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.2.0.tgz#6802e6264efd18c790a1b0d517f0f2627bf2c94a" +archiver-utils@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/archiver-utils/-/archiver-utils-1.3.0.tgz#e50b4c09c70bf3d680e32ff1b7994e9f9d895174" + dependencies: + glob "^7.0.0" + graceful-fs "^4.1.0" + lazystream "^1.0.0" + lodash "^4.8.0" + normalize-path "^2.0.0" + readable-stream "^2.0.0" + +archiver@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/archiver/-/archiver-2.1.1.tgz#ff662b4a78201494a3ee544d3a33fe7496509ebc" + dependencies: + archiver-utils "^1.3.0" + async "^2.0.0" + buffer-crc32 "^0.2.1" + glob "^7.0.0" + lodash "^4.8.0" + readable-stream "^2.0.0" + tar-stream "^1.5.0" + zip-stream "^1.2.0" + archy@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/archy/-/archy-1.0.0.tgz#f9c8c13757cc1dd7bc379ac77b2c62a5c2868c40" @@ -611,7 +635,7 @@ async@^1.4.0, async@^1.5.2: version "1.5.2" resolved "https://registry.yarnpkg.com/async/-/async-1.5.2.tgz#ec6a61ae56480c0c3cb241c95618e20892f9672a" -async@^2.1.2, async@^2.1.4, async@^2.4.1, async@~2.6.0: +async@^2.0.0, async@^2.1.2, async@^2.1.4, async@^2.4.1, async@~2.6.0: version "2.6.0" resolved "https://registry.yarnpkg.com/async/-/async-2.6.0.tgz#61a29abb6fcc026fea77e56d1c6ec53a795951f4" dependencies: @@ -1583,7 +1607,7 @@ bson@~1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/bson/-/bson-1.0.4.tgz#93c10d39eaa5b58415cbc4052f3e53e562b0b72c" -buffer-crc32@~0.2.3: +buffer-crc32@^0.2.1, buffer-crc32@~0.2.3: version "0.2.13" resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242" @@ -2181,6 +2205,15 @@ component-emitter@^1.2.0, component-emitter@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/component-emitter/-/component-emitter-1.2.1.tgz#137918d6d78283f7df7a6b7c5a63e140e69425e6" +compress-commons@^1.2.0: + version "1.2.2" + resolved "https://registry.yarnpkg.com/compress-commons/-/compress-commons-1.2.2.tgz#524a9f10903f3a813389b0225d27c48bb751890f" + dependencies: + buffer-crc32 "^0.2.1" + crc32-stream "^2.0.0" + normalize-path "^2.0.0" + readable-stream "^2.0.0" + compressible@~2.0.11: version "2.0.11" resolved "https://registry.yarnpkg.com/compressible/-/compressible-2.0.11.tgz#16718a75de283ed8e604041625a2064586797d8a" @@ -2411,6 +2444,17 @@ cosmiconfig@^4.0.0: parse-json "^4.0.0" require-from-string "^2.0.1" +crc32-stream@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/crc32-stream/-/crc32-stream-2.0.0.tgz#e3cdd3b4df3168dd74e3de3fbbcb7b297fe908f4" + dependencies: + crc "^3.4.4" + readable-stream "^2.0.0" + +crc@^3.4.4: + version "3.5.0" + resolved "https://registry.yarnpkg.com/crc/-/crc-3.5.0.tgz#98b8ba7d489665ba3979f59b21381374101a1964" + create-ecdh@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/create-ecdh/-/create-ecdh-4.0.0.tgz#888c723596cdf7612f6498233eebd7a35301737d" @@ -2641,6 +2685,12 @@ cssom@0.3.x, "cssom@>= 0.3.0 < 0.4.0", "cssom@>= 0.3.2 < 0.4.0": dependencies: cssom "0.3.x" +csv-stringify@^2.0.4: + version "2.0.4" + resolved "https://registry.yarnpkg.com/csv-stringify/-/csv-stringify-2.0.4.tgz#9ace220df98ffa7ca91314ac77ff8cba0a08c863" + dependencies: + lodash.get "~4.4.2" + cuid@~1.3.8: version "1.3.8" resolved "https://registry.yarnpkg.com/cuid/-/cuid-1.3.8.tgz#4b875e0969bad764f7ec0706cf44f5fb0831f6b7" @@ -4297,7 +4347,7 @@ got@^6.7.1: unzip-response "^2.0.1" url-parse-lax "^1.0.0" -graceful-fs@^4.1.11, graceful-fs@^4.1.2, graceful-fs@^4.1.3, graceful-fs@^4.1.4, graceful-fs@^4.1.6, graceful-fs@^4.1.9: +graceful-fs@^4.1.0, graceful-fs@^4.1.11, graceful-fs@^4.1.2, graceful-fs@^4.1.3, graceful-fs@^4.1.4, graceful-fs@^4.1.6, graceful-fs@^4.1.9: version "4.1.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" @@ -6289,6 +6339,12 @@ lazy-cache@^2.0.2: dependencies: set-getter "^0.1.0" +lazystream@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/lazystream/-/lazystream-1.0.0.tgz#f6995fe0f820392f61396be89462407bb77168e4" + dependencies: + readable-stream "^2.0.5" + lcid@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/lcid/-/lcid-1.0.0.tgz#308accafa0bc483a3867b4b6f2b9506251d1b835" @@ -6613,7 +6669,7 @@ lodash.foreach@^4.5.0: version "4.5.0" resolved "https://registry.yarnpkg.com/lodash.foreach/-/lodash.foreach-4.5.0.tgz#1a6a35eace401280c7f06dddec35165ab27e3e53" -lodash.get@4.4.2, lodash.get@^4.4.2: +lodash.get@4.4.2, lodash.get@^4.4.2, lodash.get@~4.4.2: version "4.4.2" resolved "https://registry.yarnpkg.com/lodash.get/-/lodash.get-4.4.2.tgz#2d177f652fa31e939b4438d5341499dfa3825e99" @@ -6753,7 +6809,7 @@ lodash.values@^4.3.0: version "4.3.0" resolved "https://registry.yarnpkg.com/lodash.values/-/lodash.values-4.3.0.tgz#a3a6c2b0ebecc5c2cba1c17e6e620fe81b53d347" -"lodash@>=3.5 <5", lodash@^4.13.1, lodash@^4.15.0, lodash@^4.17.5, lodash@^4.3.0, lodash@~4.17.4: +"lodash@>=3.5 <5", lodash@^4.13.1, lodash@^4.15.0, lodash@^4.17.5, lodash@^4.3.0, lodash@^4.8.0, lodash@~4.17.4: version "4.17.5" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.5.tgz#99a92d65c0272debe8c96b6057bc8fbfa3bed511" @@ -10792,6 +10848,15 @@ tar-stream@1.5.2, tar-stream@^1.1.2: readable-stream "^2.0.0" xtend "^4.0.0" +tar-stream@^1.5.0: + version "1.5.5" + resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-1.5.5.tgz#5cad84779f45c83b1f2508d96b09d88c7218af55" + dependencies: + bl "^1.0.0" + end-of-stream "^1.0.0" + readable-stream "^2.0.0" + xtend "^4.0.0" + tar@^2.0.0, tar@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/tar/-/tar-2.2.1.tgz#8e4d2a256c0e2185c6b18ad694aec968b83cb1d1" @@ -11859,3 +11924,12 @@ yauzl@^2.5.0: zen-observable-ts@^0.4.4: version "0.4.4" resolved "https://registry.yarnpkg.com/zen-observable-ts/-/zen-observable-ts-0.4.4.tgz#c244c71eaebef79a985ccf9895bc90307a6e9712" + +zip-stream@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/zip-stream/-/zip-stream-1.2.0.tgz#a8bc45f4c1b49699c6b90198baacaacdbcd4ba04" + dependencies: + archiver-utils "^1.3.0" + compress-commons "^1.2.0" + lodash "^4.8.0" + readable-stream "^2.0.0"