From 6cb1098b40fffd24c65cabd5c234497e7c784293 Mon Sep 17 00:00:00 2001 From: Denys Konovalov Date: Tue, 6 Jun 2023 17:45:05 +0200 Subject: [PATCH] feat: Gitea backend refactoring (#833) --- packages/core/src/backends/gitea/API.ts | 129 ++++++------ .../src/backends/gitea/AuthenticationPage.tsx | 47 +++-- .../src/backends/gitea/__tests__/API.spec.ts | 185 ++++++++++++------ .../src/backends/gitea/implementation.tsx | 11 +- packages/core/src/backends/gitea/types.ts | 37 ++-- packages/core/src/interface.ts | 9 +- packages/core/src/lib/auth/pkce-oauth.ts | 26 ++- .../docs/content/docs/backends-overview.mdx | 2 +- packages/docs/content/docs/gitea-backend.mdx | 20 +- 9 files changed, 293 insertions(+), 173 deletions(-) diff --git a/packages/core/src/backends/gitea/API.ts b/packages/core/src/backends/gitea/API.ts index aa8ed8aa..a7fe4783 100644 --- a/packages/core/src/backends/gitea/API.ts +++ b/packages/core/src/backends/gitea/API.ts @@ -1,9 +1,5 @@ import { Base64 } from 'js-base64'; -import initial from 'lodash/initial'; -import last from 'lodash/last'; -import partial from 'lodash/partial'; -import result from 'lodash/result'; -import trim from 'lodash/trim'; +import { trimStart, trim, result, partial, last, initial } from 'lodash'; import { APIError, @@ -22,12 +18,12 @@ import type { ApiRequest, FetchError } from '@staticcms/core/lib/util'; import type AssetProxy from '@staticcms/core/valueObjects/AssetProxy'; import type { Semaphore } from 'semaphore'; import type { + FilesResponse, GitGetBlobResponse, GitGetTreeResponse, GiteaUser, ReposGetResponse, ReposListCommitsResponse, - ContentsResponse, } from './types'; export const API_NAME = 'Gitea'; @@ -40,6 +36,20 @@ export interface Config { originRepo?: string; } +enum FileOperation { + CREATE = 'create', + DELETE = 'delete', + UPDATE = 'update', +} + +export interface ChangeFileOperation { + content?: string; + from_path?: string; + path: string; + operation: FileOperation; + sha?: string; +} + interface MetaDataObjects { entry: { path: string; sha: string }; files: MediaFile[]; @@ -76,13 +86,6 @@ type MediaFile = { path: string; }; -export type Diff = { - path: string; - newFile: boolean; - sha: string; - binary: boolean; -}; - export default class API { apiRoot: string; token: string; @@ -120,7 +123,7 @@ export default class API { static DEFAULT_COMMIT_MESSAGE = 'Automatically generated by Static CMS'; - user(): Promise<{ full_name: string; login: string }> { + user(): Promise<{ full_name: string; login: string; avatar_url: string }> { if (!this._userPromise) { this._userPromise = this.getUser(); } @@ -365,50 +368,53 @@ export default class API { async persistFiles(dataFiles: DataFile[], mediaFiles: AssetProxy[], options: PersistOptions) { // eslint-disable-next-line @typescript-eslint/no-explicit-any const files: (DataFile | AssetProxy)[] = mediaFiles.concat(dataFiles as any); - for (const file of files) { - const item: { raw?: string; sha?: string; toBase64?: () => Promise } = file; - const contentBase64 = await result( - item, - 'toBase64', - partial(this.toBase64, item.raw as string), - ); - try { - const oldSha = await this.getFileSha(file.path); - await this.updateBlob(contentBase64, file, options, oldSha!); - } catch { - await this.createBlob(contentBase64, file, options); - } - } + const operations = await this.getChangeFileOperations(files, this.branch); + return this.changeFiles(operations, options); } - async updateBlob( - contentBase64: string, - file: AssetProxy | DataFile, - options: PersistOptions, - oldSha: string, - ) { - await this.request(`${this.repoURL}/contents/${file.path}`, { - method: 'PUT', - body: JSON.stringify({ - branch: this.branch, - content: contentBase64, - message: options.commitMessage, - sha: oldSha, - signoff: false, - }), - }); - } - - async createBlob(contentBase64: string, file: AssetProxy | DataFile, options: PersistOptions) { - await this.request(`${this.repoURL}/contents/${file.path}`, { + async changeFiles(operations: ChangeFileOperation[], options: PersistOptions) { + return (await this.request(`${this.repoURL}/contents`, { method: 'POST', body: JSON.stringify({ branch: this.branch, - content: contentBase64, + files: operations, message: options.commitMessage, - signoff: false, }), - }); + })) as FilesResponse; + } + + async getChangeFileOperations(files: { path: string; newPath?: string }[], branch: string) { + const items: ChangeFileOperation[] = await Promise.all( + files.map(async file => { + const content = await result( + file, + 'toBase64', + partial(this.toBase64, (file as DataFile).raw), + ); + let sha; + let operation; + let from_path; + let path = trimStart(file.path, '/'); + try { + sha = await this.getFileSha(file.path, { branch }); + operation = FileOperation.UPDATE; + from_path = file.newPath && path; + path = file.newPath ? trimStart(file.newPath, '/') : path; + } catch { + sha = undefined; + operation = FileOperation.CREATE; + } + + return { + operation, + content, + path, + from_path, + sha, + } as ChangeFileOperation; + }), + ); + return items; } async getFileSha(path: string, { repoURL = this.repoURL, branch = this.branch } = {}) { @@ -434,15 +440,18 @@ export default class API { } async deleteFiles(paths: string[], message: string) { - for (const file of paths) { - const meta: ContentsResponse = await this.request(`${this.repoURL}/contents/${file}`, { - method: 'GET', - }); - await this.request(`${this.repoURL}/contents/${file}`, { - method: 'DELETE', - body: JSON.stringify({ branch: this.branch, message, sha: meta.sha, signoff: false }), - }); - } + const operations: ChangeFileOperation[] = await Promise.all( + paths.map(async path => { + const sha = await this.getFileSha(path); + + return { + operation: FileOperation.DELETE, + path, + sha, + } as ChangeFileOperation; + }), + ); + return this.changeFiles(operations, { commitMessage: message }); } toBase64(str: string) { diff --git a/packages/core/src/backends/gitea/AuthenticationPage.tsx b/packages/core/src/backends/gitea/AuthenticationPage.tsx index 6233064a..04822c42 100644 --- a/packages/core/src/backends/gitea/AuthenticationPage.tsx +++ b/packages/core/src/backends/gitea/AuthenticationPage.tsx @@ -1,8 +1,8 @@ import { Gitea as GiteaIcon } from '@styled-icons/simple-icons/Gitea'; -import React, { useCallback, useState } from 'react'; +import React, { useCallback, useMemo, useState } from 'react'; import Login from '@staticcms/core/components/login/Login'; -import { NetlifyAuthenticator } from '@staticcms/core/lib/auth'; +import { PkceAuthenticator } from '@staticcms/core/lib/auth'; import type { AuthenticationPageProps, TranslatedProps } from '@staticcms/core/interface'; import type { MouseEvent } from 'react'; @@ -10,36 +10,45 @@ import type { MouseEvent } from 'react'; const GiteaAuthenticationPage = ({ inProgress = false, config, - base_url, - siteId, - authEndpoint, + clearHash, onLogin, t, }: TranslatedProps) => { const [loginError, setLoginError] = useState(null); + const auth = useMemo(() => { + const { base_url = 'https://try.gitea.io', app_id = '' } = config.backend; + + const clientSizeAuth = new PkceAuthenticator({ + base_url, + auth_endpoint: 'login/oauth/authorize', + app_id, + auth_token_endpoint: 'login/oauth/access_token', + clearHash, + }); + + // Complete authentication if we were redirected back to from the provider. + clientSizeAuth.completeAuth((err, data) => { + if (err) { + setLoginError(err.toString()); + } else if (data) { + onLogin(data); + } + }); + return clientSizeAuth; + }, [clearHash, config.backend, onLogin]); + const handleLogin = useCallback( (e: MouseEvent) => { e.preventDefault(); - const cfg = { - base_url, - site_id: document.location.host.split(':')[0] === 'localhost' ? 'cms.netlify.com' : siteId, - auth_endpoint: authEndpoint, - }; - const auth = new NetlifyAuthenticator(cfg); - - const { auth_scope: authScope = '' } = config.backend; - - const scope = authScope || 'repo'; - auth.authenticate({ provider: 'gitea', scope }, (err, data) => { + auth.authenticate({ scope: 'repository' }, err => { if (err) { setLoginError(err.toString()); - } else if (data) { - onLogin(data); + return; } }); }, - [authEndpoint, base_url, config.backend, onLogin, siteId], + [auth], ); return ( diff --git a/packages/core/src/backends/gitea/__tests__/API.spec.ts b/packages/core/src/backends/gitea/__tests__/API.spec.ts index 9355cd10..2bf103c2 100644 --- a/packages/core/src/backends/gitea/__tests__/API.spec.ts +++ b/packages/core/src/backends/gitea/__tests__/API.spec.ts @@ -101,12 +101,24 @@ describe('gitea API', () => { }); describe('persistFiles', () => { - it('should check if file exists and create a new file', async () => { + it('should create a new commit', async () => { const api = new API({ branch: 'master', repo: 'owner/repo' }); const responses = { - '/repos/owner/repo/contents/content/posts/new-post.md': () => ({ + '/repos/owner/repo/git/trees/master:content%2Fposts': () => { + return { tree: [{ path: 'update-post.md', sha: 'old-sha' }] }; + }, + + '/repos/owner/repo/contents': () => ({ commit: { sha: 'new-sha' }, + files: [ + { + path: 'content/posts/new-post.md', + }, + { + path: 'content/posts/update-post.md', + }, + ], }), }; mockAPI(api, responses); @@ -115,85 +127,142 @@ describe('gitea API', () => { dataFiles: [ { slug: 'entry', - sha: 'abc', path: 'content/posts/new-post.md', raw: 'content', }, - ], - assets: [], - }; - await api.persistFiles(entry.dataFiles, entry.assets, { - commitMessage: 'commitMessage', - newEntry: true, - }); - - expect(api.request).toHaveBeenCalledTimes(2); - - expect((api.request as jest.Mock).mock.calls[0]).toEqual([ - '/repos/owner/repo/git/trees/master:content%2Fposts', - ]); - - expect((api.request as jest.Mock).mock.calls[1]).toEqual([ - '/repos/owner/repo/contents/content/posts/new-post.md', - { - method: 'POST', - body: JSON.stringify({ - branch: 'master', - content: Base64.encode(entry.dataFiles[0].raw), - message: 'commitMessage', - signoff: false, - }), - }, - ]); - }); - it('should get the file sha and update the file', async () => { - jest.clearAllMocks(); - const api = new API({ branch: 'master', repo: 'owner/repo' }); - - const responses = { - '/repos/owner/repo/git/trees/master:content%2Fposts': () => { - return { tree: [{ path: 'update-post.md', sha: 'old-sha' }] }; - }, - - '/repos/owner/repo/contents/content/posts/update-post.md': () => { - return { commit: { sha: 'updated-sha' } }; - }, - }; - mockAPI(api, responses); - - const entry = { - dataFiles: [ { slug: 'entry', - sha: 'abc', + sha: 'old-sha', path: 'content/posts/update-post.md', raw: 'content', }, ], assets: [], }; - - await api.persistFiles(entry.dataFiles, entry.assets, { - commitMessage: 'commitMessage', - newEntry: false, + await expect( + api.persistFiles(entry.dataFiles, entry.assets, { + commitMessage: 'commitMessage', + newEntry: true, + }), + ).resolves.toEqual({ + commit: { sha: 'new-sha' }, + files: [ + { + path: 'content/posts/new-post.md', + }, + { + path: 'content/posts/update-post.md', + }, + ], }); - expect(api.request).toHaveBeenCalledTimes(2); + expect(api.request).toHaveBeenCalledTimes(3); expect((api.request as jest.Mock).mock.calls[0]).toEqual([ '/repos/owner/repo/git/trees/master:content%2Fposts', ]); expect((api.request as jest.Mock).mock.calls[1]).toEqual([ - '/repos/owner/repo/contents/content/posts/update-post.md', + '/repos/owner/repo/git/trees/master:content%2Fposts', + ]); + + expect((api.request as jest.Mock).mock.calls[2]).toEqual([ + '/repos/owner/repo/contents', { - method: 'PUT', + method: 'POST', body: JSON.stringify({ branch: 'master', - content: Base64.encode(entry.dataFiles[0].raw), + files: [ + { + operation: 'create', + content: Base64.encode(entry.dataFiles[0].raw), + path: entry.dataFiles[0].path, + }, + { + operation: 'update', + content: Base64.encode(entry.dataFiles[1].raw), + path: entry.dataFiles[1].path, + sha: entry.dataFiles[1].sha, + }, + ], + message: 'commitMessage', + }), + }, + ]); + }); + }); + + describe('deleteFiles', () => { + it('should check if files exist and delete them', async () => { + const api = new API({ branch: 'master', repo: 'owner/repo' }); + + const responses = { + '/repos/owner/repo/git/trees/master:content%2Fposts': () => { + return { + tree: [ + { path: 'delete-post-1.md', sha: 'old-sha-1' }, + { path: 'delete-post-2.md', sha: 'old-sha-2' }, + ], + }; + }, + + '/repos/owner/repo/contents': () => ({ + commit: { sha: 'new-sha' }, + files: [ + { + path: 'content/posts/delete-post-1.md', + }, + { + path: 'content/posts/delete-post-2.md', + }, + ], + }), + }; + mockAPI(api, responses); + + const deleteFiles = ['content/posts/delete-post-1.md', 'content/posts/delete-post-2.md']; + + await expect(api.deleteFiles(deleteFiles, 'commitMessage')).resolves.toEqual({ + commit: { sha: 'new-sha' }, + files: [ + { + path: 'content/posts/delete-post-1.md', + }, + { + path: 'content/posts/delete-post-2.md', + }, + ], + }); + + expect(api.request).toHaveBeenCalledTimes(3); + + expect((api.request as jest.Mock).mock.calls[0]).toEqual([ + '/repos/owner/repo/git/trees/master:content%2Fposts', + ]); + + expect((api.request as jest.Mock).mock.calls[1]).toEqual([ + '/repos/owner/repo/git/trees/master:content%2Fposts', + ]); + + expect((api.request as jest.Mock).mock.calls[2]).toEqual([ + '/repos/owner/repo/contents', + { + method: 'POST', + body: JSON.stringify({ + branch: 'master', + files: [ + { + operation: 'delete', + path: deleteFiles[0], + sha: 'old-sha-1', + }, + { + operation: 'delete', + path: deleteFiles[1], + sha: 'old-sha-2', + }, + ], message: 'commitMessage', - sha: 'old-sha', - signoff: false, }), }, ]); diff --git a/packages/core/src/backends/gitea/implementation.tsx b/packages/core/src/backends/gitea/implementation.tsx index cb0afa42..79a4f601 100644 --- a/packages/core/src/backends/gitea/implementation.tsx +++ b/packages/core/src/backends/gitea/implementation.tsx @@ -77,7 +77,7 @@ export default class Gitea implements BackendClass { this.api = this.options.API || null; this.repo = this.originRepo = config.backend.repo || ''; this.branch = config.backend.branch?.trim() || 'main'; - this.apiRoot = config.backend.api_root || 'https://gitea.com/api/v1'; + this.apiRoot = config.backend.api_root || 'https://try.gitea.io/api/v1'; this.token = ''; this.mediaFolder = config.media_folder; this.lock = asyncLock(); @@ -173,8 +173,15 @@ export default class Gitea implements BackendClass { throw new Error('Your Gitea user account does not have access to this repo.'); } + console.log(user); + // Authorized user - return { ...user, token: state.token as string }; + return { + name: user.full_name, + login: user.login, + avatar_url: user.avatar_url, + token: state.token as string, + }; } logout() { diff --git a/packages/core/src/backends/gitea/types.ts b/packages/core/src/backends/gitea/types.ts index a64f8b37..9156a738 100644 --- a/packages/core/src/backends/gitea/types.ts +++ b/packages/core/src/backends/gitea/types.ts @@ -46,13 +46,13 @@ export type GiteaOrganization = { website: string; }; -type ReposListCommitsResponseItemCommitUser = { +type CommitUser = { date: string; email: string; name: string; }; -type ReposListCommitsResponseItemCommitMeta = { +type CommitMeta = { created: string; sha: string; url: string; @@ -73,10 +73,10 @@ type PayloadCommitVerification = { }; type ReposListCommitsResponseItemCommit = { - author: ReposListCommitsResponseItemCommitUser; - committer: ReposListCommitsResponseItemCommitUser; + author: CommitUser; + committer: CommitUser; message: string; - tree: ReposListCommitsResponseItemCommitMeta; + tree: CommitMeta; url: string; verification: PayloadCommitVerification; }; @@ -183,7 +183,7 @@ type ReposListCommitsResponseItem = { created: string; files: Array; html_url: string; - parents: Array; + parents: Array; sha: string; stats: ReposListCommitsResponseItemCommitStats; url: string; @@ -217,18 +217,13 @@ export type GitGetTreeResponse = { url: string; }; -export type GiteaIdentity = { - email: string; - name: string; -}; - type FileLinksResponse = { git: string; html: string; self: string; }; -export type ContentsResponse = { +type ContentsResponse = { _links: FileLinksResponse; content?: string | null; download_url: string; @@ -245,3 +240,21 @@ export type ContentsResponse = { type: string; url: string; }; + +type FileCommitResponse = { + author: CommitUser; + committer: CommitUser; + created: string; + html_url: string; + message: string; + parents: Array; + sha: string; + tree: CommitMeta; + url: string; +}; + +export type FilesResponse = { + commit: FileCommitResponse; + content: Array; + verification: PayloadCommitVerification; +}; diff --git a/packages/core/src/interface.ts b/packages/core/src/interface.ts index 25507940..10112a18 100644 --- a/packages/core/src/interface.ts +++ b/packages/core/src/interface.ts @@ -570,7 +570,14 @@ export interface MediaLibraryConfig { folder_support?: boolean; } -export type BackendType = 'git-gateway' | 'github' | 'gitlab' | 'bitbucket' | 'test-repo' | 'proxy'; +export type BackendType = + | 'git-gateway' + | 'github' + | 'gitlab' + | 'gitea' + | 'bitbucket' + | 'test-repo' + | 'proxy'; export type MapWidgetType = 'Point' | 'LineString' | 'Polygon'; diff --git a/packages/core/src/lib/auth/pkce-oauth.ts b/packages/core/src/lib/auth/pkce-oauth.ts index 6944e351..dc426de5 100644 --- a/packages/core/src/lib/auth/pkce-oauth.ts +++ b/packages/core/src/lib/auth/pkce-oauth.ts @@ -120,19 +120,25 @@ export default class PkceAuthenticator { if ('code' in params) { const code = params.code; const authURL = new URL(this.auth_token_url); - authURL.searchParams.set('client_id', this.appID); - authURL.searchParams.set('code', code ?? ''); - authURL.searchParams.set('grant_type', 'authorization_code'); - authURL.searchParams.set( - 'redirect_uri', - document.location.origin + document.location.pathname, - ); - authURL.searchParams.set('code_verifier', getCodeVerifier() ?? ''); + + const response = await fetch(authURL.href, { + method: 'POST', + body: JSON.stringify({ + client_id: this.appID, + code: code ?? '', + grant_type: 'authorization_code', + redirect_uri: document.location.origin + document.location.pathname, + code_verifier: getCodeVerifier() ?? '', + }), + headers: { + 'Content-Type': 'application/json; charset=utf-8', + }, + }); + const data = await response.json(); + //no need for verifier code so remove clearCodeVerifier(); - const response = await fetch(authURL.href, { method: 'POST' }); - const data = await response.json(); cb(null, { token: data.access_token, ...data }); } } diff --git a/packages/docs/content/docs/backends-overview.mdx b/packages/docs/content/docs/backends-overview.mdx index 55a6d5a0..14015c78 100644 --- a/packages/docs/content/docs/backends-overview.mdx +++ b/packages/docs/content/docs/backends-overview.mdx @@ -17,7 +17,7 @@ Individual backends provide their own configuration documentation, but there are | branch | string | `main` | _Optional_. The branch where published content is stored. All CMS commits and PRs are made to this branch | | api_root | string | GitHub
`https://api.github.com`

GitLab
`https://gitlab.com/api/v4`

Bitbucket
`https://api.bitbucket.org/2.0`

Gitea
`https://try.gitea.io/api/v1` | _Optional_. The API endpoint. Only necessary in certain cases, like with GitHub Enterprise or self-hosted GitLab | | site_domain | string | `location.hostname`

On `localhost`
`cms.netlify.com` | _Optional_. Sets the `site_id` query param sent to the API endpoint. Non-Netlify auth setups will often need to set this for local development to work properly | -| base_url | string | GitHub or Bitbucket
`https://api.netlify.com`

GitLab
`https://gitlab.com` | _Optional_. OAuth client hostname (just the base domain, no path). **Required** when using an external OAuth server or self-hosted GitLab/Gitea | +| base_url | string | GitHub or Bitbucket
`https://api.netlify.com`

GitLab
`https://gitlab.com`

Gitea
`https://try.gitea.io` | _Optional_. OAuth client hostname (just the base domain, no path). **Required** when using an external OAuth server or self-hosted GitLab/Gitea | | auth_endpoint | string | GitHub or Bitbucket
`auth`

GitLab
`oauth/authorize` | _Optional_. Path to append to `base_url` for authentication requests. | ## Creating a New Backend diff --git a/packages/docs/content/docs/gitea-backend.mdx b/packages/docs/content/docs/gitea-backend.mdx index a851d7cd..21d42587 100644 --- a/packages/docs/content/docs/gitea-backend.mdx +++ b/packages/docs/content/docs/gitea-backend.mdx @@ -9,24 +9,23 @@ beta: true For repositories stored on Gitea, the `gitea` backend allows CMS users to log in directly with their Gitea account. Note that all users must have push access to your content repository for this to work. -Because of the [lack](https://github.com/go-gitea/gitea/issues/14619) of a Gitea API endpoint for multifile commits, when using this backend, separate commits are created for every changed file. Please make sure this is handled correctly by your CI. +Please note that only Gitea **1.20** and upwards is supported due to API limitations in previous versions. ## Authentication -Because Gitea requires a server for authentication and Netlify doesn't support Gitea, a custom OAuth provider needs to be used for basic Gitea authentication. +With Gitea's PKCE authorization, users can authenticate with Gitea directly from the client. To do this: -To enable basic Gitea authentication: - -1. Setup an own OAuth provider, for example with [Teabag](https://github.com/denyskon/teabag). -2. Add the following lines to your Static CMS `config` file: +1. Add your Static CMS instance as an OAuth application in your user/organization settings or through the admin panel of your Gitea instance. Please make sure to uncheck the **Confidential Client** checkbox. For the **Redirect URIs**, enter the addresses where you access Static CMS, for example, `https://www.mysite.com/admin/`. +2. Gitea provides you with a **Client ID**. Copy it and insert it into your `config` file along with the other options: ```yaml backend: name: gitea repo: owner-name/repo-name # Path to your Gitea repository - base_url: https://oauth.example.com # URL of your OAuth provider - api_root: https://gitea.example.com/api/v1 # API url of your Gitea instance + app_id: your-client-id # The Client ID provided by Gitea + api_root: https://gitea.example.com/api/v1 # API URL of your Gitea instance + base_url: https://gitea.example.com # Root URL of your Gitea instance # optional, defaults to main # branch: main ``` @@ -35,8 +34,9 @@ backend: backend: { name: 'gitea', repo: 'owner-name/repo-name', // Path to your Gitea repository - base_url: 'https://oauth.example.com', // URL of your OAuth provider - api_root: 'https://gitea.example.com/api/v1' // API url of your Gitea instance + app_id: 'your-client-id', // The Client ID provided by Gitea + api_root: 'https://gitea.example.com/api/v1', // API URL of your Gitea instance + base_url: 'https://gitea.example.com', // Root URL of your Gitea instance // optional, defaults to main // branch: 'main' },