ethcon.ubq.fi and ethcon-banner.ubq.fi

[0x4007]

We have some swag bags we gave out with QR codes that go to those URLs. The banner is not on display these days but I figured we would fix it at the same time. Currently the QR codes are broken (they go to ethcon.ubq.fi)

We have a system that dynamically will link to our subdomains based on the repository name. For example, if I create a new repository called test.ubq.fi our continuous deploys script will automatically 1. create test-ubq-fi.pages.dev and 2. link our subdomain to it so that its immediately accessible on the internet from our subdomain.

The core magic is inside of this Cloudflare Worker logic:

addEventListener('fetch', event => {
  event.respondWith(handleRequest(event.request))
})

async function handleRequest(request) {
  const url = new URL(request.url)
  let address
  
  if (url.hostname === 'ubq.fi') {
    // Route root domain
    address = 'https://ubq-fi.pages.dev' + url.pathname
  } else {
    // Route subdomains
    const subdomain = url.hostname.split('.')[0]
    address = `https://${subdomain}-ubq-fi.pages.dev` + url.pathname
  }

  return fetch(address, request)
}

We should have some type of exceptions capability for simple forwarding. Perhaps a key/value map.

{
  "ethcon.ubq.fi": "https://dao.ubq.fi/ethcon",
  "ethcon-banner.ubq.fi": "https://dao.ubq.fi/ethcon-banner",
}

This might be simple enough to just post the code as a comment here or something. I can manually test it and then we can complete the task. You really should attempt to test the code though, even if its substitute code that does the same thing.

[ubiquibot]

! action has an uncaught error

[0xJoichiro]

@0x4007 do you mean to add this key,value mapping as an exception case value and have an extra case checking for the same in the if/else statement?

[0x4007]

and have an extra case checking for the same in the if/else statement?

Should be dynamically handled

[0xJoichiro]

Yes i understand the same being dynamically handled,where can i get this worker code to test it for myself?

Should be dynamically handled

[0x4007]

I don't understand your question. You write the code. Use what I included as a starting point.

[PhantomCracker]

Hello @0x4007 , I would like to ask if the provided code is in this repository because I can't find it, or I should look on another one?
Thanks.

[0x4007]

The code is in the specification

[0xJoichiro]

I wrote a small crawler for finding simple forwarding links,can we just dynamically update these keys everytime cloudflare worker script runs for the new repositories/event for subdomain.We could then use the results from crawler script as an exception case.Some of them are listed below

/dollar
/devpool
/the-mission
/investors
/the-team
/the-devpool
/the-labs
/earn-stake
/branding
/events
/recruiting

[0xJoichiro]

The code is in the specification

should i just add the crawler script that makes the simple forwarding key/value pairs here in comments?

[0xJoichiro]

const fs = require('fs');
const http = require('http');
const https = require('https');

const queue = [];
const crawledUrls = [];
const cleanedUrls = [];
const urlMap = {};

const initialUrl = 'https://dao.ubq.fi/';

async function crawlWebsite(url) {

    return new Promise((resolve, reject) => {

        const protocol = url.startsWith('https') ? https : http;

        protocol.get(url, response => {
            let data = '';

            response.on('data', chunk => {
                data += chunk;
            });

            response.on('end', () => {
                const links = getDataLinks(data);

                links.forEach(link => {

                    var href = link.match(/href=["']([^"']*)["']/)[1];

                    // Prepend the protocol and domain if the link is relative
                    if (!href.startsWith('http')) {
                        href = url.slice(0, url.indexOf('://') + 3) + href;
                    }

                    if (isInternalLink(link, url)) {
                    }
                    else {

                        if (href.startsWith('https:///')) {
                            const url = href;

                            // Split the URL by "//" and get the second part
                            const parts = url.split('//');

                            const pathWithLeadingSlash = parts[1];
                            // Remove the leading slash from the path
                            const extractedPath = pathWithLeadingSlash.replace(/^\//, '');

                            if (extractedPath == "") return;
                            addToQueue(getFullUrl(extractedPath, href))
                            // queue.push(getFullUrl(extractedPath, href));
                            crawledUrls.push(getFullUrl(extractedPath, href));
                            resolve();

                        }
                    }
                });

                saveCrawledUrls();

            });
        });
    })

}

function addToQueue(url) {
    if (!queue.includes(url)) {
        queue.push(url);
    }
}

function getDataLinks(data) {
    const linkRegex = /<a[^>]*href=["']([^"']*)["']/gi;
    return data.match(linkRegex);
}

function isInternalLink(href, parentUrl) {
    return (href.startsWith('https://' + parentUrl.hostname + '/') && href !== 'https://' + parentUrl.hostname + '/')
}

function getFullUrl(link, parentUrl) {
    return `${initialUrl}${link}`;
}

function getBaseUrl(url) {
    return url.slice(0, url.indexOf('/', url.indexOf('://') + 3));
}

function saveCrawledUrls() {
    const newUrls = crawledUrls.filter(url => !urlMap[url]);

    newUrls.forEach(url => {
        urlMap[url] = true;
        cleanedUrls.push(url);
    });

    fs.writeFileSync('crawled_urls.json', JSON.stringify(cleanedUrls));
}

async function init() {
    queue.push(initialUrl);

    while (queue.length > 0) {

        console.log("queue.length", queue.length)
        const url = queue.shift();
        await crawlWebsite(url);
    }

}

init();

this is my crawler script to find all the links and we can have a short script for making a mapping as suggested for simple forwarding,does this work?

[0xJoichiro]

@0x4007 sample output in json was this(includes ethcon and ethcon-banner) =>

["https://dao.ubq.fi/dollar","https://dao.ubq.fi/devpool","https://dao.ubq.fi/about","https://dao.ubq.fi/announcements","https://dao.ubq.fi/overview","https://dao.ubq.fi/docs","https://dao.ubq.fi/communication-guidelines","https://dao.ubq.fi/devpool-flow","https://dao.ubq.fi/devpool-flow","https://dao.ubq.fi/devpool-appendix","https://dao.ubq.fi/the-mission","https://dao.ubq.fi/investors","https://dao.ubq.fi/the-team","https://dao.ubq.fi/the-devpool","https://dao.ubq.fi/the-labs","https://dao.ubq.fi/earn-stake","https://dao.ubq.fi/branding","https://dao.ubq.fi/events","https://dao.ubq.fi/recruiting","https://dao.ubq.fi/investor-updates","https://dao.ubq.fi/announcements-archive","https://dao.ubq.fi/smart-contracts","https://dao.ubq.fi/tokenomics","https://dao.ubq.fi/devpool-landing-page","https://dao.ubq.fi/ubiquibot-setup","https://dao.ubq.fi/non-code-devpool-deliverables-process","https://dao.ubq.fi/devpool-pinned-message","https://dao.ubq.fi/scaling-web3-projects-development","https://dao.ubq.fi/optimism-grant-iii-basic-data","https://dao.ubq.fi/ubiquibot-qa","https://dao.ubq.fi/animoca-ventures","https://dao.ubq.fi/concave","https://dao.ubq.fi/merit-circle","https://dao.ubq.fi/momentum6","https://dao.ubq.fi/ascensive-assets","https://dao.ubq.fi/small-cap-scientist","https://dao.ubq.fi/snackclub","https://dao.ubq.fi/c-ventures","https://dao.ubq.fi/play-ventures","https://dao.ubq.fi/grills","https://dao.ubq.fi/coud","https://dao.ubq.fi/dcf-god","https://dao.ubq.fi/369-capital","https://dao.ubq.fi/new-tribe-capital","https://dao.ubq.fi/balthazar","https://dao.ubq.fi/アレクサンダーeth","https://dao.ubq.fi/hodlatoor","https://dao.ubq.fi/hristo-piyankov","https://dao.ubq.fi/ser","https://dao.ubq.fi/zgo","https://dao.ubq.fi/zequez","https://dao.ubq.fi/zapaz","https://dao.ubq.fi/0xcodercrane","https://dao.ubq.fi/rndquu","https://dao.ubq.fi/hashedmae","https://dao.ubq.fi/steveantor","https://dao.ubq.fi/draeieg","https://dao.ubq.fi/akarin","https://dao.ubq.fi/niamie","https://dao.ubq.fi/nate-the-noble","https://dao.ubq.fi/thxforplayin","https://dao.ubq.fi/chefyaboyardi","https://dao.ubq.fi/nicholasnokia","https://dao.ubq.fi/elsepatura","https://dao.ubq.fi/nangolilla","https://dao.ubq.fi/bentj8w7","https://dao.ubq.fi/xeno","https://dao.ubq.fi/nnova","https://dao.ubq.fi/0xrickjames","https://dao.ubq.fi/ubiquity-dao-logo","https://dao.ubq.fi/ubiquity-dollar-token-icon","https://dao.ubq.fi/ubiquity-governance-token-icon","https://dao.ubq.fi/ubiquity-credit-token-icon","https://dao.ubq.fi/logo-lockup-black","https://dao.ubq.fi/logo-lockup-white","https://dao.ubq.fi/ethcon","https://dao.ubq.fi/ethcon-banner","https://dao.ubq.fi/q4-2023-recruiting","https://dao.ubq.fi/q2-2024-recruiting","https://dao.ubq.fi/q2-2022","https://dao.ubq.fi/q4-2022","https://dao.ubq.fi/q2-2023","https://dao.ubq.fi/q4-2023","https://dao.ubq.fi/prelaunch","https://dao.ubq.fi/press-releases","https://dao.ubq.fi/23-june-2021","https://dao.ubq.fi/28-june-2021","https://dao.ubq.fi/29-june-2021","https://dao.ubq.fi/1-july-2021","https://dao.ubq.fi/16-july-2021","https://dao.ubq.fi/23-july-2021","https://dao.ubq.fi/24-july-2021","https://dao.ubq.fi/bonding-v2-pre-migration-notice","https://dao.ubq.fi/31-july-2021","https://dao.ubq.fi/bonding-v2-pre-migration-notice-follow-up","https://dao.ubq.fi/ubq-bbq-1","https://dao.ubq.fi/bondingv2-migration-update-1","https://dao.ubq.fi/ubq-rewards-program","https://dao.ubq.fi/bondingv2-migration-update-2","https://dao.ubq.fi/bonding-v21-launched","https://dao.ubq.fi/to-migrate-or-not","https://dao.ubq.fi/q4-plans--august-update","https://dao.ubq.fi/introducing-the-ubiquity-proxy-yield-aggregator","https://dao.ubq.fi/airdrop-proxy-yield-aggregator-new-team-members-partnerships-and-development-news","https://dao.ubq.fi/targeted-incentivization","https://dao.ubq.fi/uad-transfer-hook-announcement-partnerships-integrations-and-other-development-news","https://dao.ubq.fi/the-ubiquity-proxy-yield-aggregator-beta-islive","https://dao.ubq.fi/ubiquity-dao-is-now-stepping-on-the-metaverse","https://dao.ubq.fi/nft-drop-perks-and-more","https://dao.ubq.fi/partnership-memos","https://dao.ubq.fi/holiday-craft-2023","https://dao.ubq.fi/55af4c60d3c843d997a8017a44c1ce4d","https://dao.ubq.fi/bonding-v2-upgrade","https://dao.ubq.fi/bonding-v2-upgrade","https://dao.ubq.fi/dollar-collateralization","https://dao.ubq.fi/token-list"]

[0xJoichiro]

here is the mapping function

function mapping() {

    const cleanUrlsFromFile = JSON.parse(fs.readFileSync('crawled_urls.json'));
    const daoUrls = [];

    for (const url of cleanUrlsFromFile) {
        if (url.startsWith('https://dao.ubq.fi/')) {
            const path = url.split('https://dao.ubq.fi/')[1];
            daoUrls.push(path);
        }
    }

    cleanUrlsFromFile.forEach((element, i) => {
        mappingUrls[daoUrls[i] + '.ubq.fi'] = (element);
    });
    return mappingUrls;
}

[0xJoichiro]

/start

[0xJoichiro]

Should i make a PR for this? please redirect me to where this code needs to be written and testes.

Thank you for your time