From cac32298b8d4b558b9b170b5cb90986f58b57161 Mon Sep 17 00:00:00 2001 From: Eryn Wells Date: Sat, 23 Sep 2023 09:52:33 -0700 Subject: [PATCH 1/2] Bones of a Twitter section Copied over the templates from blog/ and set up permalinks. Drew some bespoke star and retweet icons. Wrote a script to import data from the archive. --- assets/styles/twitter.css | 49 +++++ config/_default/permalinks.yaml | 1 + layouts/twitter/li_grid_with_date.html | 7 + layouts/twitter/list.html | 26 +++ layouts/twitter/section.atom.atom | 0 layouts/twitter/section.rss.rss | 0 layouts/twitter/single.html | 15 ++ scripts/import-twitter-archive.js | 264 +++++++++++++++++++++++++ static/images/retweet.png | Bin 0 -> 1264 bytes static/images/star-unfilled.png | Bin 0 -> 1781 bytes 10 files changed, 362 insertions(+) create mode 100644 assets/styles/twitter.css create mode 100644 layouts/twitter/li_grid_with_date.html create mode 100644 layouts/twitter/list.html create mode 100644 layouts/twitter/section.atom.atom create mode 100644 layouts/twitter/section.rss.rss create mode 100644 layouts/twitter/single.html create mode 100755 scripts/import-twitter-archive.js create mode 100644 static/images/retweet.png create mode 100644 static/images/star-unfilled.png diff --git a/assets/styles/twitter.css b/assets/styles/twitter.css new file mode 100644 index 0000000..d0387c6 --- /dev/null +++ b/assets/styles/twitter.css @@ -0,0 +1,49 @@ +:root { + --tweet-separator-color: rgb(var(--lt-gray)); +} + +article.tweet { + border: 1px solid var(--tweet-separator-color); + border-radius: 6px; + padding: 4rem; +} + +ul.tweets { + list-style-type: none; + margin-inline-start: 0; +} + +li.tweet { + border-bottom: 1px solid var(--tweet-separator-color); + margin: 0; + padding-block: 2rem; +} + +li.tweet:first-of-type { + border-top: 1px solid var(--tweet-separator-color); +} + +.metadata { + display: flex; + list-style-type: none; + margin: 0; + margin-block-start: 1rem; +} + +.metadata .timestamp { + color: rgb(var(--mid-gray)); + font-size: 90%; + margin: 0; +} + +.metadata :is(.rt, .fav) { + gap: 0.5rem; + display: flex; + align-items: center; + position: relative; +} + +.metadata .rt img { + position: relative; + top: 1px; +} diff --git a/config/_default/permalinks.yaml b/config/_default/permalinks.yaml index 8efe757..7d4b0bc 100644 --- a/config/_default/permalinks.yaml +++ b/config/_default/permalinks.yaml @@ -1,2 +1,3 @@ blog: blog/:year/:month/:slug/ photos: photos/:year/:month/:slug/ +twitter: twitter/:year/:month/:slug/ diff --git a/layouts/twitter/li_grid_with_date.html b/layouts/twitter/li_grid_with_date.html new file mode 100644 index 0000000..a58625a --- /dev/null +++ b/layouts/twitter/li_grid_with_date.html @@ -0,0 +1,7 @@ +
  • + {{ .Content }} + + {{ partial "development/draft_tag.html" . }} +
    • diff --git a/layouts/twitter/list.html b/layouts/twitter/list.html new file mode 100644 index 0000000..0d5d420 --- /dev/null +++ b/layouts/twitter/list.html @@ -0,0 +1,26 @@ +{{ define "header" }} + {{ partial "header.html" . }} +{{ end }} + +{{ define "main" }} +

    {{ .Title }}

    + + {{ .Content }} + + {{ $paginator := .Paginator 100 }} + {{ template "_internal/pagination.html" . }} + + + + {{ template "_internal/pagination.html" . }} +{{ end }} + +{{ define "footer" }} + {{ partial "footer.html" . }} +{{ end }} diff --git a/layouts/twitter/section.atom.atom b/layouts/twitter/section.atom.atom new file mode 100644 index 0000000..e69de29 diff --git a/layouts/twitter/section.rss.rss b/layouts/twitter/section.rss.rss new file mode 100644 index 0000000..e69de29 diff --git a/layouts/twitter/single.html b/layouts/twitter/single.html new file mode 100644 index 0000000..bde1ecc --- /dev/null +++ b/layouts/twitter/single.html @@ -0,0 +1,15 @@ +{{ define "main" }} +
    + {{ .Content }} +
    +
      +
    • + {{ .Date | time.Format ":date_full" }}, + {{ .Date | time.Format ":time_short" }} +
      • +
    • {{ .Params.tweet.tweet.favorite_count }}
      • +
    • {{ .Params.tweet.tweet.retweet_count }}
      • +
    +
    +
    +{{ end }} diff --git a/scripts/import-twitter-archive.js b/scripts/import-twitter-archive.js new file mode 100755 index 0000000..0f82141 --- /dev/null +++ b/scripts/import-twitter-archive.js @@ -0,0 +1,264 @@ +#!/usr/bin/env node + +"use strict"; + +const fsPromises = require("node:fs/promises"); +const path = require("node:path"); + +class TweetProcessingError extends Error { + tweet; + + constructor(tweet, message) { + super(`[${tweet.id}]: ${message}`); + this.tweet = tweet; + } +} + +class MediaEntity { + tweet; + entity; + + fileName; + mediaFilePath; + + constructor(tweet, entity) { + this.tweet = tweet; + this.entity = entity; + } + + process(mediaPath) { + const url = new URL(this.entity.media_url); + this.fileName = path.basename(url.pathname); + this.mediaFilePath = path.join(mediaPath, `${this.tweet.id}-${this.fileName}`); + } + + async copyMediaFile(contentDirectory) { + const destinationPath = path.join(contentDirectory, this.fileName); + console.log(`[${this.tweet.id}]: Copying media file from ${this.mediaFilePath} to ${destinationPath}`); + + try { + await fsPromises.copyFile(this.mediaFilePath, destinationPath); + } catch (error) { + console.error(error); + throw error; + } + } +} + +class Tweet { + tweet; + mediaEntities = []; + + constructor(tweet) { + this.tweet = tweet; + } + + get id() { + return this.#innerTweet.id_str; + } + + get dateCreated() { + return new Date(this.#innerTweet.created_at); + } + + get text() { + return this.#innerTweet.full_text; + } + + contentPath(asDirectory = true) { + const innerTweet = this.#innerTweet; + + const idString = innerTweet.id_str; + const timestamp = this.dateCreated; + + return path.join( + timestamp.getFullYear().toString().padStart(4, '0'), + // Months are 0-11. + (timestamp.getMonth() + 1).toString().padStart(2, '0'), + asDirectory ? idString : `${idString}.md` + ); + } + + get #innerTweet() { return this.tweet.tweet; } + + async process(hugoContentPath, mediaPath) { + const dateCreated = this.dateCreated; + const indexObject = { + date: dateCreated.toISOString(), + title: this.text, + slug: this.id, + tweet: this.tweet, + }; + + + let textOfTweet = this.text; + let references = []; + + let doTextSubstitution = (startIndex, endIndex, replacementText) => { + const textBefore = textOfTweet.substring(0, startIndex); + const textAfter = textOfTweet.substring(endIndex); + textOfTweet = `${textBefore}${replacementText}${textAfter}`; + }; + + let processTextEntitySubstitution = (_, item, i) => { + const [startIndex, endIndex] = item.indices; + const reference = `[${textOfTweet.substring(startIndex, endIndex)}][entity${i}]`; + doTextSubstitution(startIndex, endIndex, reference); + + references.push(`[entity${i}]: https://twitter.com/${item.screen_name}`); + }; + + let processMediaEntitySubstitution = (_, item, __) => { + const entity = new MediaEntity(this, item); + this.mediaEntities.push(entity); + entity.process(mediaPath); + + const [startIndex, endIndex] = item.indices; + const reference = `\n\n{{< figures/image name="${entity.fileName}" >}}\n\n`; + doTextSubstitution(startIndex, endIndex, reference); + }; + + let shouldMakeDirectory = false; + + Object.entries(this.#innerTweet.entities) + .map(([entityType, entities]) => entities.map(e => [entityType, e])) + .flatMap(e => e) + .sort(([, a], [, b]) => { + const startIndexOfA = a.indices[0]; + const startIndexOfB = b.indices[0]; + + if (startIndexOfA === startIndexOfB) { + console.assert(false); + return 0; + } + + // Reverse sort by start index of the entity. + return startIndexOfA < startIndexOfB ? 1 : -1; + }).forEach(([typeOfItem, item], i) => { + switch (typeOfItem) { + case "hashtags": + case "user_mentions": + processTextEntitySubstitution(typeOfItem, item, i); + break; + case "media": + shouldMakeDirectory = true; + processMediaEntitySubstitution(typeOfItem, item, i); + break; + case "symbols": + // Symbols appear to be stock ticker symbols. They may be other things. + console.log(`[${this.id}]: Encountered symbol entity. Ignoring.`, item); + break; + } + }); + + + const frontMatter = JSON.stringify(indexObject, null, 4); + const joinedReferences = references.reverse().join("\n"); + let contentsOfIndexFile = `${frontMatter}\n\n${textOfTweet}\n\n${joinedReferences}`; + + let tweetFilePath; + if (shouldMakeDirectory) { + const contentPath = path.join(hugoContentPath, this.contentPath()); + tweetFilePath = path.join(contentPath, "index.md"); + + console.log(`[${this.id}]: Writing tweet file to ${tweetFilePath}`); + await fsPromises.mkdir(contentPath, { recursive: true }); + + for (const mediaEntity of this.mediaEntities) { + await mediaEntity.copyMediaFile(contentPath); + } + } else { + tweetFilePath = path.join(hugoContentPath, this.contentPath(false)); + const containingDirectory = path.dirname(tweetFilePath); + + console.log(`[${this.id}]: Writing tweet file to ${tweetFilePath}`); + await fsPromises.mkdir(containingDirectory, { recursive: true }); + } + + try { + await fsPromises.writeFile(tweetFilePath, contentsOfIndexFile); + } catch (error) { + console.error(error); + throw error; + } + + return this; + } +} + +function* take(iterable, length) { + const iterator = iterable[Symbol.iterator](); + while (length-- > 0) { + yield iterator.next().value; + } +} + +function* readTweets(tweetsJSONPath) { + tweetsJSONPath = path.resolve(tweetsJSONPath); + + console.log("Loading tweets from", tweetsJSONPath); + const tweets = require(tweetsJSONPath); + + for (const tweetObject of tweets) { + yield new Promise(resolve => { + resolve(new Tweet(tweetObject)); + }); + } +} + +async function main() { + const programArguments = process.argv; + + // The first two arguments are always `node` and then the script. + if (programArguments.length !== 4) { + console.error("Invalid number of program arguments. Expected 3."); + return -1; + } + + const twitterArchivePath = programArguments[2]; + const hugoContentPath = programArguments[3]; + + try { + await fsPromises.access(twitterArchivePath); + } catch (error) { + console.error(error); + return -1; + } + + const twitterArchiveDataPath = path.join(twitterArchivePath, "data"); + try { + await fsPromises.access(twitterArchiveDataPath); + } catch (error) { + console.error(`${twitterArchivePath} doesn't appear to be a valid Twitter archive. It's missing a data directory!`); + console.error(error); + return -1; + } + + try { + await fsPromises.access(hugoContentPath); + } catch (error) { + console.error(error); + return -1; + } + + const tweetsFilePath = path.join(twitterArchiveDataPath, "tweets.json"); + const tweetsMediaPath = path.join(twitterArchiveDataPath, "tweets_media"); + + let numberOfTweetsProcessed = 0; + + await Promise.all( + [...take(readTweets(tweetsFilePath), 1000)].map(tweetPromise => { + return tweetPromise + .then(tweet => { + numberOfTweetsProcessed++; + return tweet.process(hugoContentPath, tweetsMediaPath); + }) + .catch(error => console.error(error)); + })); + + console.log(`\nSuccessfully processed ${numberOfTweetsProcessed} tweets.`); + + return 0; +} + +main(); diff --git a/static/images/retweet.png b/static/images/retweet.png new file mode 100644 index 0000000000000000000000000000000000000000..61af3a9669e44c94a3824dbafe08c41fd7151c85 GIT binary patch literal 1264 zcmV+0$< zAt%F%;!T`}8CZp{abY7BwweHQGN0$82g?lh=2~er0a8rC3UuSrS|4r)0ZLqkjd%`| zt9_`Q1b|L-;d@+B%>(TwK#7a70WXOmQkHfVfN+XSu?bg~yQ!T8C~*P4!mA>FIEwKg z08GT=_z~9)y-?TOiIc@RJ6p`nvvC|w#5AGwjKTj?V{ppgwQ?N6-*wtniSw`?tML?i zYZ2fYEW{l+H-FoXVuBEGwCa8YXX37E1h@e!Mby&6yr!|$Ted+=|AMpmNU-J5>ACc@+cLocKRn1dV3RP;Wc6^(8ceim=NJ9RN7 z!0pn$;X^!-pIxI)znHyVkREbK3sBDEug<}!qS%ZF@xw?rrrvZa$T=*Qn9WX~*bkVe zipG=f0jDO)cF8!8CZtEa^)0nLluZYic~{9*~;;_SRbQD6w6{ z@Fh0jA#AO;!c-4P?WZ-%>B$o85Iyn$rhyIfPG^9yeMyr3MLiLOwSe^bAn@Sz*uuk zfE?iD?q16OzIKaPeVDQ z&N}lZb)YkBQCH9D@LUlm%&p_uBAJyaJDe@3r3XwF2PPXef8$B_#9i5fYJ0$ZtPx|r z6*yhm2mF>T7^w$L#VaBsvsD@ocb6&Olr5;#1FjV-sPpr%s30cbesLO<;)`rSN`Pr1 zy=;l*4U{^?v^xuDiPV?NakH3;Q~Zf_*}|*<*W+!G`_l-iQ=~q%og7cZ2}3@2isSQ{ z5{sHDSS2wFlmwVA#`Z7GztVnvY+k0lFGgW|J1G!%h?5rPjw z*x&+0VvLUjjY2IDBSs7hgXl&R6F?y>EN`TkC<{UHl}PNu049Wh5?WiZkn*`W=l1)0 znVIj-eQCl!xykp;Xm^mgEL%RL{LiH$)y)KQ0Y=`bz(_ z?%z}LzXx+P7>c-1X2_ZVTNgHs{I+OdbD>m_B?0x6AN@fY->b@nE0u_{zk^aHNfG=*tfXSb-!d zvxIGs3u6!NUXliyfajq|z%v*d za8PAHR&Q7J3YMx}Kp$|0;#=-SM&V$T_Q4JYMwv<^8i6dY2Ln~WFfFS`#<6OOYlmUL z4B$~q9i31l;0x?-Fdf(reB{}PG*Fr{`^)p$mk`@+XoI2)tf3e$!>9r_0G}dnB3I%G~DwwqFI{fLqe=l~u- z>jzOIJeH@LPblvB%#es8ZcqFz+*f z*tQO_waX$i06rO^YrraGo6EMSsQ^|YZ=k_LcR4hu&Xw2?Sac&v*j0$Fkq{>(wjGJN zCmk}gvCqNdU?%1aQ-G6*O*?!Q$IS`;giM+aTkiKEBXn9gh&?19R$G3Kqc}-Y^iSY9 z%e`Yr`YA13KxTO+ELU4%A!$~4hNXOeLcJ!~&Jfych=imB%(Rqeap2LWb5;_iNeCDM zJYp!nfVe=~u({R4YD+96Az&)zcTZxW0R<$7 zSPp}LR^X6cYl(JT=K$uJXzV#hTmnV|6Ab0Q0B6cPI|hl}tC8t#74Qb|mELpVjMfpx z9mLF3#U;Sj{hcc>U;#3Tnryh&sP`6>(P672mK&r^fGz&l0-KSkYJ)>>gUW2izTmOd z5z7tY5Kx7A^3e?(F_ce0a&XH$>R6`FnB43L=A&~nh_MQdL%@R+Hz`Wdj%d!0&Ig# z;G0vB)cSmfjsox=GVnDbn;5RIx8?&sQcQu_#sTkv(3gPQfx8Xm9mqyQ3lid7;p{we zeC}oB6h|-e0aY{ZjhWF%*MC#%GNrrjQ5z$}mw+bAGbc)na}7iXfjcpWvVFi^n5Vk| zswk5DtKohnp648V39w~l9><-by^5r>W&vj%a%WX$C84uCAxuH?em*uD;Y+}Mp7H>{ zBNLx@h?c9nrm~auSzmbB=u3dF{8l7~f3%#S0Y~-Ot)9<$+34G#sCl<;sg^!>u5qk>3OZcU2SUdRakMOe?I>& XgW6RtQeR~t00000NkvXXu0mjft$;Iw literal 0 HcmV?d00001 From 47cc68482f309b71d7e3f1b473fb43e48da50e70 Mon Sep 17 00:00:00 2001 From: Eryn Wells Date: Sat, 23 Sep 2023 09:52:46 -0700 Subject: [PATCH 2/2] Styles for the Hugo internal pagination controls --- assets/styles/root/050_pagination.css | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 assets/styles/root/050_pagination.css diff --git a/assets/styles/root/050_pagination.css b/assets/styles/root/050_pagination.css new file mode 100644 index 0000000..d48d5de --- /dev/null +++ b/assets/styles/root/050_pagination.css @@ -0,0 +1,25 @@ +ul.pagination { + display: flex; + margin-inline: 0; + justify-content: center; +} + +ul.pagination > li { + list-style-type: none; +} + +.page-item.active a { + color: var(--text-color); +} + +.page-item.active a:hover { + text-decoration: none; +} + +.page-item.disabled a { + color: rgb(var(--mid-gray)); +} + +.page-item.disabled a:hover { + text-decoration: none; +}