diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b993c23..1848917 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,17 +5,16 @@ name: Release # существующим git-тегом v) — тогда: # 1. Публикуем workspace-пакеты в npm # 2. Создаём git-тег v и пушим его -# 3. Создаём GitHub Release с changelog'ом (changelogithub) -# 4. Собираем и пушим Docker-образ в docker.io (Docker Hub) +# 3. Создаём Gitea Release с changelog'ом через Gitea API # Если версия не менялась — workflow no-op (без ошибки). # # Требуемые secrets в настройках репозитория: -# DOCKERHUB_USERNAME — Docker Hub логин с доступом к организации dicoop -# DOCKERHUB_TOKEN — PAT с правами Read,Write из hub.docker.com/settings/security -# GITHUB_TOKEN — выдаётся автоматически GitHub Actions -# -# npm publish работает через trusted publishing (OIDC) — токен не нужен: -# настроен на стороне npm для @coopenomics/parser2 под этот workflow. +# NPM_TOKEN — npm access token с правами publish на @coopenomics/* +# (Gitea Actions не выдаёт OIDC id-token, поэтому npm +# trusted publishing/provenance здесь недоступны — +# используем классический auth через NODE_AUTH_TOKEN) +# GITHUB_TOKEN — выдаётся автоматически Gitea Actions; используется для +# создания релиза через ${GITHUB_SERVER_URL}/api/v1/... on: push: @@ -26,8 +25,7 @@ concurrency: cancel-in-progress: false permissions: - contents: write # tag + GitHub Release - id-token: write # npm trusted publishing (OIDC) + sigstore provenance + contents: write # tag + Gitea Release jobs: release: @@ -39,7 +37,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - fetch-depth: 0 # нужно для git tag проверки и changelogithub + fetch-depth: 0 # нужно для git tag проверки и git log changelog - uses: pnpm/action-setup@v4 with: { version: '10' } - uses: actions/setup-node@v4 @@ -71,15 +69,15 @@ jobs: pnpm --filter @coopenomics/coopos-ship-reader build pnpm --filter @coopenomics/parser2 build - # npm trusted publishing: OIDC через id-token: write, без NPM_TOKEN. - # Настроено на стороне npm: @coopenomics/parser2/access → Trusted Publisher - # → GitHub Actions (repo=coopenomics/parser2, workflow=release.yml). - # Provenance=true — sigstore подпишет каждый publish. + # Auth через NODE_AUTH_TOKEN: actions/setup-node по registry-url прописал + # //registry.npmjs.org/:_authToken=${NODE_AUTH_TOKEN} в ~/.npmrc, npm/lerna + # читают токен из env. Provenance не включаем — требует OIDC id-token, + # которого Gitea Actions не выдаёт (Trusted Publisher тоже не сработает). - name: Publish via Lerna (from-package) if: steps.ver.outputs.released == 'true' run: pnpm exec lerna publish from-package --yes --no-private env: - NPM_CONFIG_PROVENANCE: true + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Tag release @@ -90,52 +88,29 @@ jobs: git tag "v${{ steps.ver.outputs.version }}" git push origin "v${{ steps.ver.outputs.version }}" - - name: Create GitHub Release + # changelogithub знает только github.com и падает на git.coopenomics.world + # ("Can not parse GitHub repo from url ..."). Создаём релиз напрямую через + # Gitea REST API: GITHUB_SERVER_URL и GITHUB_REPOSITORY проставляются + # раннером, GITHUB_TOKEN имеет права contents:write на тот же репо. + # Changelog — git log между предыдущим v-тегом и текущим. + - name: Create Gitea Release if: steps.ver.outputs.released == 'true' - run: npx changelogithub + run: | + VERSION="${{ steps.ver.outputs.version }}" + PREV=$(git tag --list 'v*' --sort=-v:refname | grep -vx "v${VERSION}" | head -1) + if [ -n "$PREV" ]; then + CHANGELOG=$(git log "${PREV}..v${VERSION}" --pretty=format:'- %s') + else + CHANGELOG=$(git log "v${VERSION}" --pretty=format:'- %s') + fi + BODY=$(jq -n \ + --arg tag "v${VERSION}" \ + --arg body "$CHANGELOG" \ + '{tag_name:$tag, name:$tag, body:$body, draft:false, prerelease:false}') + curl -fsS -X POST \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "$BODY" \ + "${GITHUB_SERVER_URL}/api/v1/repos/${GITHUB_REPOSITORY}/releases" env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - docker: - name: Build & push Docker image - runs-on: ubuntu-22.04 - needs: release - if: needs.release.outputs.released == 'true' - steps: - - uses: actions/checkout@v4 - with: - ref: v${{ needs.release.outputs.version }} - - uses: pnpm/action-setup@v4 - with: { version: '10' } - - uses: actions/setup-node@v4 - with: { node-version: '20', cache: 'pnpm' } - - run: pnpm install --frozen-lockfile - - run: pnpm --filter @coopenomics/coopos-ship-reader build - - run: pnpm --filter @coopenomics/parser2 build - - - uses: docker/setup-buildx-action@v3 - # Docker Hub auth: требует secrets DOCKERHUB_USERNAME + DOCKERHUB_TOKEN - # (Access Token из hub.docker.com/settings/security с правами Read,Write) - - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: dicoop/parser2 - tags: | - type=semver,pattern={{version}},value=v${{ needs.release.outputs.version }} - type=semver,pattern={{major}}.{{minor}},value=v${{ needs.release.outputs.version }} - type=raw,value=latest - - uses: docker/build-push-action@v5 - with: - context: . - file: Dockerfile - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max - platforms: linux/amd64,linux/arm64 diff --git a/packages/parser2/package.json b/packages/parser2/package.json index 5bbc8ea..10863e2 100644 --- a/packages/parser2/package.json +++ b/packages/parser2/package.json @@ -1,6 +1,6 @@ { "name": "@coopenomics/parser2", - "version": "1.1.0", + "version": "1.1.1", "description": "Universal EOSIO/Antelope SHiP-to-Redis blockchain indexer (parser)", "license": "MIT", "author": "Coopenomics contributors", diff --git a/packages/parser2/src/adapters/ShipReaderAdapter.ts b/packages/parser2/src/adapters/ShipReaderAdapter.ts index cfc4cc1..5f1aa8e 100644 --- a/packages/parser2/src/adapters/ShipReaderAdapter.ts +++ b/packages/parser2/src/adapters/ShipReaderAdapter.ts @@ -86,6 +86,6 @@ export class ShipReaderAdapter implements ChainClient { * содержит hardcoded-схемы нативных таблиц (permission, account, …). */ deserializeNativeDelta(delta: ShipDelta): ShipNativeDeltaEvent { - return this.client.deserializer.deserializeNativeDelta(delta) + return this.client.deserializer.deserializeNativeDelta(delta, this.client.abi) } } diff --git a/packages/parser2/src/core/BlockProcessor.ts b/packages/parser2/src/core/BlockProcessor.ts index 569b696..a925338 100644 --- a/packages/parser2/src/core/BlockProcessor.ts +++ b/packages/parser2/src/core/BlockProcessor.ts @@ -29,6 +29,7 @@ import { computeEventId } from '../events/eventId.js' import type { AbiBootstrapper } from '../abi/AbiBootstrapper.js' import type { AbiStore } from '../abi/AbiStore.js' import type { ChainClient } from '../ports/ChainClient.js' +import { rootLogger, type Logger } from '../logger.js' interface BlockProcessorOptions { /** Идентификатор цепи — проставляется в каждое событие. */ @@ -68,6 +69,7 @@ export class BlockProcessor { private abiBootstrapper: AbiBootstrapper private abiStore: AbiStore private chainClient: ChainClient + private log: Logger constructor(opts: BlockProcessorOptions) { this.chainId = opts.chainId @@ -75,6 +77,7 @@ export class BlockProcessor { this.abiBootstrapper = opts.abiBootstrapper this.abiStore = opts.abiStore this.chainClient = opts.chainClient + this.log = rootLogger.child({ component: 'BlockProcessor', chain_id: opts.chainId }) this.queue = new PQueue({ concurrency: 1 }) } @@ -237,8 +240,13 @@ export class BlockProcessor { present: native.present, } nativeDeltaEvents.push({ ...partial, event_id: computeEventId(partial) }) - } catch { - // Ошибки в отдельных нативных дельтах не должны прерывать весь блок + } catch (err) { + // Ошибки в отдельных нативных дельтах не должны прерывать весь блок, + // но молча терять их нельзя — иначе регрессии десериализации невидимы. + this.log.warn( + { block_num: blockNum, table: delta.name, err: err instanceof Error ? err.message : String(err) }, + 'native delta deserialize failed', + ) } } } diff --git a/packages/ship-reader/package.json b/packages/ship-reader/package.json index c1eae21..f41e9d6 100644 --- a/packages/ship-reader/package.json +++ b/packages/ship-reader/package.json @@ -1,6 +1,6 @@ { "name": "@coopenomics/coopos-ship-reader", - "version": "0.2.0", + "version": "0.3.0", "description": "Clean-room SHiP WebSocket client for EOSIO/Antelope blockchains", "license": "MIT", "author": "Coopenomics contributors", diff --git a/packages/ship-reader/src/NativeRowStream.ts b/packages/ship-reader/src/NativeRowStream.ts index 55a0068..a4ccd21 100644 --- a/packages/ship-reader/src/NativeRowStream.ts +++ b/packages/ship-reader/src/NativeRowStream.ts @@ -1,3 +1,4 @@ +import type { ABI } from '@wharfkit/antelope' import type { ShipDelta } from './types/ship.js' import type { NativeDeltaEvent } from './native-tables/index.js' import { isNativeTableName } from './native-tables/index.js' @@ -10,9 +11,10 @@ export function filterNativeDeltas(deltas: readonly ShipDelta[]): ShipDelta[] { export function* streamNativeDeltas( deltas: readonly ShipDelta[], deserializer: WharfkitDeserializer, + abi: ABI, ): Generator { for (const delta of deltas) { if (!isNativeTableName(delta.name)) continue - yield deserializer.deserializeNativeDelta(delta) + yield deserializer.deserializeNativeDelta(delta, abi) } } diff --git a/packages/ship-reader/src/ShipClient.ts b/packages/ship-reader/src/ShipClient.ts index 93881c7..02e3929 100644 --- a/packages/ship-reader/src/ShipClient.ts +++ b/packages/ship-reader/src/ShipClient.ts @@ -21,6 +21,15 @@ export class ShipClient { this.deserializer = new WharfkitDeserializer() } + /** + * SHiP-ABI (state_history), доступен после connect(). + * Нужен для десериализации нативных дельт (их строки сериализованы этим ABI). + */ + get abi(): ShipAbi { + if (!this.shipAbi) throw new ShipConnectionError('Call connect() first') + return this.shipAbi + } + async connect(): Promise { if (this.ws?.readyState === WebSocket.OPEN) return diff --git a/packages/ship-reader/src/deserializers/Deserializer.ts b/packages/ship-reader/src/deserializers/Deserializer.ts index f2d6969..fbcc865 100644 --- a/packages/ship-reader/src/deserializers/Deserializer.ts +++ b/packages/ship-reader/src/deserializers/Deserializer.ts @@ -5,6 +5,6 @@ import type { NativeDeltaEvent } from '../native-tables/index.js' export interface Deserializer { deserializeAction>(trace: ShipTrace, abi: ABI): Action deserializeContractRow>(delta: ShipDelta, abi: ABI): Delta - deserializeNativeDelta>(delta: ShipDelta): NativeDeltaEvent + deserializeNativeDelta>(delta: ShipDelta, abi: ABI): NativeDeltaEvent readonly name: 'wharfkit' } diff --git a/packages/ship-reader/src/deserializers/WharfkitDeserializer.ts b/packages/ship-reader/src/deserializers/WharfkitDeserializer.ts index f0d571d..7a36848 100644 --- a/packages/ship-reader/src/deserializers/WharfkitDeserializer.ts +++ b/packages/ship-reader/src/deserializers/WharfkitDeserializer.ts @@ -57,13 +57,18 @@ export class WharfkitDeserializer implements Deserializer { } } - deserializeNativeDelta>(delta: ShipDelta): NativeDeltaEvent { + deserializeNativeDelta>(delta: ShipDelta, abi: ABI): NativeDeltaEvent { if (!isNativeTableName(delta.name)) { throw new UnknownNativeTableError(delta.name) } const table = delta.name as NativeTableName try { - const data = JSON.parse(Buffer.from(delta.rowRaw).toString('utf8')) as T + // rowRaw нативных таблиц — это ABI-сериализованные байты state_history, + // а НЕ JSON. Тип строки = имя таблицы: в ship-ABI это variant (*_v0). + const decoded = Serializer.decode({ data: delta.rowRaw, type: table, abi }) + const objectified = Serializer.objectify(decoded as ABISerializable) + // variant оформлен как [typeName, row] — распаковываем саму строку. + const data = (Array.isArray(objectified) ? objectified[1] : objectified) as T const lookup_key = computeLookupKey(table, data as never) const present: boolean = delta.present return { present, table, data, lookup_key } diff --git a/packages/ship-reader/test/unit/native-rows.test.ts b/packages/ship-reader/test/unit/native-rows.test.ts index 34f0d67..84a6afd 100644 --- a/packages/ship-reader/test/unit/native-rows.test.ts +++ b/packages/ship-reader/test/unit/native-rows.test.ts @@ -1,11 +1,50 @@ import { describe, it, expect } from 'vitest' +import { ABI, Serializer } from '@wharfkit/antelope' import { WharfkitDeserializer } from '../../src/deserializers/WharfkitDeserializer.js' import { computeLookupKey } from '../../src/native-tables/index.js' import { isNativeTableName, NATIVE_TABLE_NAMES } from '../../src/native-tables/types.js' -import { UnknownNativeTableError } from '../../src/errors.js' +import { UnknownNativeTableError, DeserializationError } from '../../src/errors.js' import type { ShipDelta } from '../../src/types/ship.js' import type { NativePermissionRow, NativePermissionLinkRow } from '../../src/native-tables/types.js' +/** + * Мини-ABI в стиле state_history: каждая нативная таблица — это variant (*_v0). + * Достаточно полей, нужных computeLookupKey. Кодируем и декодируем одним ABI — + * схема самосогласована, повторять реальный EOSIO целиком не требуется. + */ +function makeShipAbi(): ABI { + return ABI.from({ + version: 'eosio::abi/1.1', + types: [], + structs: [ + { name: 'permission_v0', base: '', fields: [ + { name: 'owner', type: 'name' }, + { name: 'name', type: 'name' }, + ] }, + { name: 'account_v0', base: '', fields: [ + { name: 'name', type: 'name' }, + { name: 'creation_date', type: 'string' }, + { name: 'abi', type: 'string' }, + ] }, + ], + actions: [], + tables: [], + variants: [ + { name: 'permission', types: ['permission_v0'] }, + { name: 'account', types: ['account_v0'] }, + ], + }) +} + +const SHIP_ABI = makeShipAbi() + +/** Дельта с ABI-сериализованным rowRaw (как реально приходит из SHiP). */ +function abiDelta(table: string, obj: Record, present = true): ShipDelta { + const encoded = Serializer.encode({ object: [`${table}_v0`, obj], type: table, abi: SHIP_ABI }) + return { name: table, present, rowRaw: encoded.array } +} + +/** Дельта с JSON-байтами — раньше «работала» из-за бага JSON.parse, теперь должна падать. */ function jsonDelta(name: string, data: unknown, present = true): ShipDelta { return { name, @@ -64,36 +103,38 @@ describe('computeLookupKey', () => { describe('WharfkitDeserializer — deserializeNativeDelta', () => { const deser = new WharfkitDeserializer() - it('deserializes permission delta with correct lookup_key', () => { - const permData: NativePermissionRow = { - owner: 'alice', name: 'active', parent: 'owner', - last_updated: '2024-01-01T00:00:00.000', - auth: { threshold: 1, keys: [], accounts: [], waits: [] }, - } - const delta = jsonDelta('permission', permData) - const event = deser.deserializeNativeDelta(delta) + it('deserializes ABI-encoded permission delta with correct lookup_key', () => { + const delta = abiDelta('permission', { owner: 'alice', name: 'active' }) + const event = deser.deserializeNativeDelta(delta, SHIP_ABI) expect(event.table).toBe('permission') expect(event.lookup_key).toBe('alice:active') expect(event.present).toBe(true) - expect(event.data.owner).toBe('alice') + expect(String(event.data.owner)).toBe('alice') + }) + + it('unwraps the *_v0 variant into a flat row (not a [name, row] tuple)', () => { + const delta = abiDelta('permission', { owner: 'alice', name: 'active' }) + const event = deser.deserializeNativeDelta(delta, SHIP_ABI) + expect(Array.isArray(event.data)).toBe(false) + expect(String(event.data.name)).toBe('active') }) it('present field is boolean (not string)', () => { - const delta = jsonDelta('account', { name: 'bob', creation_date: '', abi: '' }, false) - const event = deser.deserializeNativeDelta(delta) + const delta = abiDelta('account', { name: 'bob', creation_date: '2024-01-01T00:00:00.000', abi: '' }, false) + const event = deser.deserializeNativeDelta(delta, SHIP_ABI) expect(typeof event.present).toBe('boolean') expect(event.present).toBe(false) }) it('throws UnknownNativeTableError for unknown table', () => { const delta: ShipDelta = { name: 'my_custom_table', present: true, rowRaw: new Uint8Array([]) } - expect(() => deser.deserializeNativeDelta(delta)).toThrow(UnknownNativeTableError) + expect(() => deser.deserializeNativeDelta(delta, SHIP_ABI)).toThrow(UnknownNativeTableError) }) - it('all NATIVE_TABLE_NAMES tables deserialize without throwing (smoke test)', () => { - for (const table of NATIVE_TABLE_NAMES) { - const delta = jsonDelta(table, { name: 'test' }) - expect(() => deser.deserializeNativeDelta(delta)).not.toThrow() - } + it('throws DeserializationError on non-ABI (JSON) bytes — no silent JSON.parse', () => { + // Регресс-гард на исходный баг: нативные строки декодятся ABI, а не JSON.parse. + // JSON-байты — не валидная ABI-сериализация, метод обязан бросить. + const delta = jsonDelta('permission', { owner: 'alice', name: 'active' }) + expect(() => deser.deserializeNativeDelta(delta, SHIP_ABI)).toThrow(DeserializationError) }) })