Post
Topic
Board Meta
Merits 167 from 47 users
Topic OP
[Script] Imgur to TalkImg - automatically fix your broken images
by
TryNinja
on 14/05/2023, 07:38:14 UTC
⭐ Merited by vapourminer (10) ,NeuroticFish (10) ,HCP (10) ,PowerGlove (10) ,LoyceV (8) ,hosseinimr93 (8) ,Pmalek (8) ,suchmoon (8) ,dkbit98 (7) ,fillippone (6) ,tranthidung (5) ,joker_josue (5) ,krogothmanhattan (5) ,Husna QA (5) ,cygan (5) ,Heisenberg_Hunter (5) ,d5000 (5) ,babo (4) ,yahoo62278 (4) ,ETFbitcoin (3) ,FatFork (3) ,buwaytress (3) ,Halab (2) ,Eternad (2) ,Ryu_Ar1 (2) ,OgNasty (2) ,Tytanowy Janusz (2) ,decodx (1) ,John Abraham (1) ,Timelord2067 (1) ,aylabadia05 (1) ,libert19 (1) ,psycodad (1) ,BitMaxz (1) ,bullrun2020bro (1)
Imgur to TalkImg - automatically fix your broken images

Quote



Imgur images don't work anymore, so you probably have lots of posts with broken images that look like this:



For context: Imgur images suddenly became invalid?



So I created a script that:

1. Goes through all your posts looking for imgur.com direct links (.png|.jpg|.jpeg);
2. Reuploads the image to talkimg.com;
3. Edits the post with the new link.

How to use it:

1. Go to bitcointalk.org (any page).
2. Open the browser developer tools on the Console tab (Ctrl+Shift+I or F12).
3. Paste the script and press Enter.
4. Leave the page open while the script runs and, if you can, do not use the forum (not even in other tabs) to avoid rate limiting errors.

If a error shows up, please report it here and/or run the script again (the process will start over, but already updated posts will be ignored since they won't have any more imgur.com links).

Script:

- You're free to swap the API key if you have an account on TalkImg. Otherwise, there is already one provided by @joker_josue for this script.
- You can change the startPage variable if you want to start the script from a specific page (i.e the script errors at page 300 and you want to restart back from there).

Code:
(async () => {

    // options
    const startPage = 1
    const useProxy = true
    const apiKey = 'chv_AiD_124562a509c5fadffba3e15a3a31f8241855c36609c497a325396124b370b138a1d5ecda8061410b4a3478bdf26b51c5589e23d7e277a15dedda70577ca79995'

    const uploadUrl = useProxy ? 'https://proxy.ninjastic.space/?url=https://talkimg.com/api/1/upload' : 'https://talkimg.com/api/1/upload'
    const decoder = new TextDecoder('windows-1252')
    const parser = new DOMParser()
    let lastReq

    const fetchThrottled = async (url, ...rest) => {
        const timeRemaining = lastReq ? lastReq.getTime() + 1000 * 1 - new Date().getTime() : 0
        if (timeRemaining > 0) {
            await new Promise(resolve => setTimeout(resolve, timeRemaining))
        }
        lastReq = new Date()
        return await fetch(url, ...rest)
    }

    const decodeProxyImages = (html) => html.replaceAll(/img.*?src="(.*?)"\s/g, (text, imgUrl) => {
        const directImgUrl = imgUrl
            .replace(/https:\/\/ip\.bitcointalk\.org\/\?u=/, '')
            .replace(/&.*/, '')
        const decodedUrl = decodeURIComponent(directImgUrl)
        return text.replace(imgUrl, decodedUrl)
    })

    const encodeStr = (rawStr) => {
        return rawStr.replace(/[\u00A0-\u9999<>&]/g, (i) => `&#${i.charCodeAt(0)};`)
    }

    const getSesc = async () => {
        const html = await fetchThrottled('https://bitcointalk.org/more.php').then(async response => decoder.decode(await response.arrayBuffer()))
        return html.match(/https\:\/\/bitcointalk\.org\/index\.php\?action=logout;sesc=(.*?)"\>/)?.at(1)
    }

    const getQuote = async ({ topicId, postId, sesc }) => {
        const url = `https://bitcointalk.org/index.php?action=quotefast;xml;quote=${postId};topic=${topicId};sesc=${sesc}`
        const html = await fetchThrottled(url).then(async response => decoder.decode(await response.arrayBuffer()))
        const $ = parser.parseFromString(html, 'text/html')
        const quote = $.querySelector('quote').textContent
        return quote.replace(/^\[quote.*?\]/, '').replace(/\[\/quote\]$/, '').trim()
    }

    const editPost = async ({ topicId, postId, title, message, sesc }) => {
        const formData = new FormData()
        formData.append('topic', String(topicId))
        formData.append('subject', encodeStr(title))
        formData.append('message', encodeStr(message))
        formData.append('sc', sesc)
        formData.append('goback', String(1))
        const { redirected } = await fetchThrottled(`https://bitcointalk.org/index.php?action=post2;msg=${postId}`, {  method: 'POST', body: formData })
        return redirected
    }

    const getPosts = async (page) => {
        const url = `https://bitcointalk.org/index.php?action=profile;u=557798;sa=showPosts;start=${((page ?? 1) - 1) * 20}`
        const html = await fetchThrottled(url).then(async response => decoder.decode(await response.arrayBuffer()))
        const decoded = decodeProxyImages(html)
        const $ = parser.parseFromString(decoded, 'text/html')
        const postElements = [...$.querySelectorAll('table[width="85%"] table[width="100%"] tbody')]
            .filter(element => element.querySelector('.post'))
        const posts = []
        for (const postElement of postElements) {
            const titleElement  = postElement.querySelector('tr[class=titlebg2] td:nth-child(2) a:last-child')
            const title = titleElement.textContent.trim()
            const [, topicId, postId] = titleElement.getAttribute('href').match(/topic=(\d+)\.msg(\d+)/)
            const contentElement = postElement.querySelector('.post')
            const links = [...new Set(contentElement.innerHTML.match(/https:\/\/i\.imgur\.com\/.*?\.(png|jpg|jpeg)/gi))] ?? []
            posts.push({ topicId, postId, title, links })
        }

        return posts
    }

    const uploadImage = async (image) => {
        const formData = new FormData()
        formData.append('type', 'file')
        formData.append('format', 'json')
        formData.append('source', image)

        const upload = await fetchThrottled(uploadUrl, {
            method: 'POST',
            headers: { 'X-API-Key': apiKey },
            mode: 'cors',
            body: formData,
        })

        const response = await upload.json()
        if (response.status_code === 200) {
            return { url: response.image.url, deleteUrl: response.image.delete_url }
        }

        console.log('Could not upload, error:', response?.error?.message ?? response)
        return undefined
    }
    
    const html = await fetchThrottled('https://bitcointalk.org/index.php?action=profile;sa=showPosts').then(async response => response.text())
    const $ = parser.parseFromString(html, 'text/html')
    const isLast = $.querySelector('.prevnext:last-child > a.navPages') === null

    const lastPageNum = isLast ?
        Number($.querySelector('tbody > tr.catbg3 > td > b').textContent) :
        Number($.querySelector('td[colspan] a.navPages:nth-last-child(2)')?.textContent ?? 1)

    console.log('%cImgur to TalkImg - automatically fix your broken images', 'color: #fff; font-weight: bold; background-color: blue;')
    console.log('Number of Pages:', lastPageNum)

    let numberUploads = 0

    if (startPage > lastPageNum) {
        throw Error('startPage is greater than your number of pages')
    }

    for await (const page of Array.from({ length: lastPageNum - startPage + 1 }).map((_, i) => startPage + i)) {
        console.log(`--------------------\nGetting posts on page ${page}/${lastPageNum} (${Math.floor(page / lastPageNum * 100)}%)`)
        const posts = await getPosts(page).then(posts => posts.filter(post => post.links.length > 0))
        if (posts.length > 0) {
            console.log(`Found ${posts.length} posts and ${posts.flatMap(post => post.links).length} images`, posts)
        }
        for await (const post of posts) {
            const images = []
            for await (const link of post.links) {
                const image = await fetchThrottled(link).then(async response => response.blob())
                images.push(image)
            }

            const uploadedImages = []
            for await (const [index, image] of images.entries()) {
                if (numberUploads >= 30) {
                    numberUploads = 0
                    console.log('Upload API limited, waiting 1 minute...')
                    await new Promise(resolve => setTimeout(resolve, 1000 * 60))
                }

                console.log(`[${post.postId}] Uploading image...`)
                const uploaded = await uploadImage(image)
                if (uploaded?.url) {
                    numberUploads += 1
                    uploadedImages.push({ old: post.links[index], new: uploaded.url, deleteUrl: uploaded.deleteUrl })
                    console.log(`[${post.postId}] Uploaded:`, uploaded.url)
                }
            }
            
            if (uploadedImages.length > 0) {
                const sesc = await getSesc()
                const currPost = await getQuote({ topicId: post.topicId, postId: post.postId, sesc })
                let newContent = currPost

                for (const uploadedImage of uploadedImages) {
                    newContent = newContent.replaceAll(uploadedImage.old, uploadedImage.new)
                }

                console.log(`[${post.postId}] Editing post https://bitcointalk.org/index.php?topic=${post.topicId}.msg${post.postId}#msg${post.postId}`)
                const edited = await editPost({ topicId: post.topicId, postId: post.postId, title: post.title, message: newContent, sesc })

                if (!edited) {
                    console.log(`[${post.postId}] Could not edit post (maybe locked?), deleting uploaded images...`)
                    for (const uploadedImage of uploadedImages) {
                        await fetchThrottled(uploadedImage.deleteUrl, { redirect: 'manual' })
                    }
                }
            } else {
                console.log(`[${post.postId}] No images were uploaded, skiping edit...`)
            }
        }
    }

    console.log('-- Finished! --')
})()

Other notes:

- This will upload and edit all imgur.com links found on your post history (even those inside quotes you made of other people).
- Posts on locked topics can't be edited, so the recently uploaded images for them will be deleted (to save space on the talkimg server).