Skip to content

chore: use plain app id in key gen #960

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
Open

chore: use plain app id in key gen #960

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
75c1da4
use plain app id in key gen
juliusmarminge Sep 18, 2024
b281a5c
rm verifykey helper
juliusmarminge Sep 18, 2024
37a73ff
Merge branch 'main' into plain-app-id
juliusmarminge Oct 8, 2024
764ed1b
Merge branch 'main' into plain-app-id
juliusmarminge Nov 6, 2024
6376d33
rm
juliusmarminge Nov 6, 2024
badab68
rm unused test
juliusmarminge Nov 6, 2024
fcbfa02
Merge branch 'main' into plain-app-id
juliusmarminge Jan 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
272 changes: 4 additions & 268 deletions docs/src/app/(docs)/uploading-files/page.mdx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,275 +96,11 @@ Once your backend adapter has received and validated the request, you will next
need to generate presigned URLs. First, generate some file keys for the files to
be uploaded.

To generate a file key, generate a [Sqids](https://sqids.org/) of your appId
with `{ minLength: 12 }`, then concatenate this with a file seed of your choice.
To generate a file key, concatenate your appId with a file seed of your choice.
The file seed can be anything you want, but it should be unique for each file,
as well as url safe. In this example, we include a base64 encoding to ensure the
file seed is url safe, but you can do this however you want.

Although we currently only offer a JavaScript SDK, here are some reference
implementations you can use to generate valid file keys. We also plan on making
this process easier in the future.

<Note>
If you struggle to implement it for your language, you can also request one
from the [`/v7/prepareUpload` REST
endpoint](/api-reference/openapi-spec#tag/default/POST/v7/prepareUpload). Keep
in mind that this adds extra latency to your uploads.
</Note>

<Tabs tabs={["JavaScript", "Python", "PHP", "Go", "Rust"]}>
<Tab>

```ts
import SQIds, { defaultOptions } from "sqids";

function djb2(s: string) {
const h = 5381;
let i = s.length;
while (i) {
h = (h * 33) ^ s.charCodeAt(--i);
}
return (h & 0xbfffffff) | ((h >>> 1) & 0x40000000);
}

// A simple function to shuffle the alphabet for the Sqids
function shuffle(str: string, seed: string) {
const chars = str.split("");
const seedNum = djb2(seed);

let temp: string;
let j: number;
for (let i = 0; i < chars.length; i++) {
j = ((seedNum % (i + 1)) + i) % chars.length;
temp = chars[i];
chars[i] = chars[j];
chars[j] = temp;
}

return chars.join("");
}

function generateKey(appId: string, fileSeed: string) {
// Hash and Encode the parts and apiKey as sqids
const alphabet = shuffle(defaultOptions.alphabet, appId);

const encodedAppId = new SQIds({ alphabet, minLength: 12 }).encode([
Math.abs(djb2(appId)),
]);

// We use a base64 encoding here to ensure the file seed is url safe, but
// you can do this however you want
const encodedFileSeed = encodeBase64(fileSeed);

return `${encodedAppId}${encodedFileSeed}`;
}
```

</Tab>
<Tab>

```py
import math
import base64
from sqids import Sqids
from sqids.constants import DEFAULT_ALPHABET

def djb2(s: str) -> int:
h = 5381
for char in reversed(s):
h = (h * 33) ^ ord(char)
# 32-bit integer overflow
h &= 0xFFFFFFFF
h = (h & 0xBFFFFFFF) | ((h >> 1) & 0x40000000)

# Convert to signed 32-bit integer
if h >= 0x80000000:
h -= 0x100000000

return h


def shuffle(string: str, seed: str) -> str:
chars = list(string)
seed_num = djb2(seed)

for i in range(len(chars)):
j = int(math.fmod(math.fmod(seed_num, i + 1) + i, len(chars)))
chars[i], chars[j] = chars[j], chars[i]

return "".join(chars)


def generate_key(file_seed: str, app_id: str) -> str:
alphabet = shuffle(DEFAULT_ALPHABET, app_id)

encoded_app_id = Sqids(alphabet, min_length=12).encode(
[abs(djb2(app_id))]
)

return encoded_app_id + file_seed
```

</Tab>
<Tab>

```php
use Sqids\Sqids;

function djb2(string $string): int {
$h = 5381;
for ($i = strlen($string) - 1; $i >= 0; $i--) {
$char = $string[$i];
$h = ($h * 33) ^ ord($char);
// 32-bit integer overflow
$h &= 0xFFFFFFFF;
}
$h = ($h & 0xBFFFFFFF) | (($h >> 1) & 0x40000000);

// Convert to signed 32-bit integer
if ($h >= 0x80000000) {
$h -= 0x100000000;
}

return $h;
}

function shuffle_string(string $string, string $seed): string {
$chars = str_split($string);
$seed_num = djb2($seed);

for ($i = 0; $i < count($chars); $i++) {
$j = (($seed_num % ($i + 1)) + $i) % count($chars);
[$chars[$i], $chars[$j]] = [$chars[$j], $chars[$i]];
}

return implode('', $chars);
}

function generate_key(string $file_seed, string $appId): string {
$alphabet = shuffle_string(Sqids::DEFAULT_ALPHABET, $appId);
$sqids = new Sqids($alphabet, 12);

$encodedAppId = $sqids->encode(
[abs(djb2($appId))]
);

return $encodedAppId . base64_encode($file_seed);
}
```

</Tab>
<Tab>

```go
import (
"math"
"github.com/sqids/sqids-go"
)

func djb2(s string) int32 {
h := int64(5381)
for i := len(s) - 1; i >= 0; i-- {
h = (h * 33) ^ int64(s[i])
// 32-bit integer overflow
h &= 0xFFFFFFFF
}
h = (h & 0xBFFFFFFF) | ((h >> 1) & 0x40000000)

// Convert to signed 32-bit integer
if h >= 0x80000000 {
h -= 0x100000000
}

return int32(h)
}

func shuffle(input string, seed string) string {
chars := []rune(input)
seedNum := djb2(seed)

for i := 0; i < len(chars); i++ {
j := (int(seedNum)%(i+1) + i) % len(chars)
chars[i], chars[j] = chars[j], chars[i]
}

return string(chars)
}

func generateKey(fileSeed string, appId string) string {
alphabet := shuffle("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789", appId)
s, _ := sqids.New(sqids.Options{
MinLength: 12,
Alphabet: alphabet,
})

encodedAppId, _ := s.Encode(
[]uint64{uint64(math.Abs(float64(djb2(appId))))},
)

return encodedAppId + fileSeed
}
```

</Tab>
<Tab>

```rust
fn djb2_hash(s: &str) -> i32 {
let mut h: i64 = 5381;
for &byte in s.as_bytes().iter().rev() {
h = (h * 33) ^ (byte as i64);
// Simulate 32-bit integer overflow
h &= 0xFFFFFFFF;
}
// Convert to signed 32-bit integer with the same bit manipulation
h = (h & 0xBFFFFFFF) | ((h >> 1) & 0x40000000);
if h >= 0x80000000 {
h -= 0x100000000;
}
h as i32
}

fn shuffle(input: &str, seed: &str) -> String {
let mut chars: Vec<char> = input.chars().collect();
let seed_num = djb2_hash(seed);
for i in 0..chars.len() {
let j = ((seed_num % (i as i32 + 1)) + i as i32) as usize % chars.len();
let temp = chars[i];
chars[i] = chars[j];
chars[j] = temp;
}
chars.iter().collect()
}
fn generate_file_key(app_id: String) -> String {
let app_hash = djb2_hash(&app_id);
let alphabet: Vec<char> = shuffle(sqids::DEFAULT_ALPHABET, &app_id).chars().collect();

// https://sqids.org/rust
let sqids = sqids::Sqids::builder()
.alphabet(alphabet)
.min_length(12)
.build()
.expect("Could not create sqid builder");
let encoded_app_id = sqids.encode(&vec![app_hash.abs() as u64]).expect("Could not encode sqid");

// https://github.com/uuid-rs/uuid
let file_seed = uuid::Uuid::new_v4().to_string();

// We use a base64 encoding here to ensure the file seed is url safe, but
// you can do this however you want
// https://github.com/marshallpierce/rust-base64
use base64::prelude::*;
let encoded_file_seed = BASE64_URL_SAFE.encode(file_seed.as_bytes());

format!("{}{}", encoded_app_id, encoded_file_seed)
}

```

</Tab>
</Tabs>
url safe and no less than 36 characters. In this example, we include a base64
encoding to ensure the file seed is url safe, but you can do this however you
want.

The URL, to which you will upload the file, will depend on your app's region.
You can find the list of regions in the
Expand Down
19 changes: 1 addition & 18 deletions packages/shared/src/crypto.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,28 +111,11 @@ export const generateKey = (
const encodedFileSeed = new SQIds({ alphabet, minLength: 36 }).encode([
Math.abs(Hash.string(hashParts)),
]);
const encodedAppId = new SQIds({ alphabet, minLength: 12 }).encode([
Math.abs(Hash.string(appId)),
]);

// Concatenate them
return encodedAppId + encodedFileSeed;
return appId + encodedFileSeed;
}).pipe(Micro.withTrace("generateKey"));

// Verify that the key was generated with the same appId
export const verifyKey = (key: string, appId: string) =>
Micro.sync(() => {
const alphabet = shuffle(defaultOptions.alphabet, appId);
const expectedPrefix = new SQIds({ alphabet, minLength: 12 }).encode([
Math.abs(Hash.string(appId)),
]);

return key.startsWith(expectedPrefix);
}).pipe(
Micro.withTrace("verifyKey"),
Micro.orElseSucceed(() => false),
);

export const generateSignedURL = (
url: string | URL,
secretKey: Redacted.Redacted<string>,
Expand Down
Loading
Loading