Merge pull request #815 from cosmos/add-block-search-simon

Add support for /block_search RPC endpoint in Tendermint 0.34.9+

CHANGELOG.md

@@ -6,6 +6,18 @@ and this project adheres to
 
 ## [Unreleased]
 
+### Added
+
+- @cosmjs/tendermint-rpc: `Tendermint34Client.blockSearch` and
+  `Tendermint34Client.blockSearchAll` were added to allow searching blocks in
+  Tendermint 0.34.9+ backends.
+
+### Changes
+
+- @cosmjs/tendermint-rpc: Make `tendermint34.Header.lastBlockId` and
+  `tendermint34.Block.lastCommit` optional to better handle the case of height 1
+  where there is no previous block.
+
 ## [0.25.3] - 2021-05-18
 
 ### Fixed

packages/stargate/src/queries/queryclient.ts

@@ -2,7 +2,7 @@
 import { iavlSpec, ics23, tendermintSpec, verifyExistence, verifyNonExistence } from "@confio/ics23";
 import { toAscii, toHex } from "@cosmjs/encoding";
 import { firstEvent } from "@cosmjs/stream";
-import { Header, NewBlockHeaderEvent, ProofOp, Tendermint34Client } from "@cosmjs/tendermint-rpc";
+import { tendermint34, Tendermint34Client } from "@cosmjs/tendermint-rpc";
 import { arrayContentEquals, assert, assertDefined, isNonNullObject, sleep } from "@cosmjs/utils";
 import { Stream } from "xstream";
 
@@ -10,7 +10,7 @@ import { ProofOps } from "../codec/tendermint/crypto/proof";
 
 type QueryExtensionSetup<P> = (base: QueryClient) => P;
 
-function checkAndParseOp(op: ProofOp, kind: string, key: Uint8Array): ics23.CommitmentProof {
+function checkAndParseOp(op: tendermint34.ProofOp, kind: string, key: Uint8Array): ics23.CommitmentProof {
   if (op.type !== kind) {
     throw new Error(`Op expected to be ${kind}, got "${op.type}`);
   }
@@ -587,15 +587,15 @@ export class QueryClient {
 
   // this must return the header for height+1
   // throws an error if height is 0 or undefined
-  private async getNextHeader(height?: number): Promise<Header> {
+  private async getNextHeader(height?: number): Promise<tendermint34.Header> {
     assertDefined(height);
     if (height === 0) {
       throw new Error("Query returned height 0, cannot prove it");
     }
 
     const searchHeight = height + 1;
-    let nextHeader: Header | undefined;
-    let headersSubscription: Stream<NewBlockHeaderEvent> | undefined;
+    let nextHeader: tendermint34.Header | undefined;
+    let headersSubscription: Stream<tendermint34.NewBlockHeaderEvent> | undefined;
     try {
       headersSubscription = this.tmClient.subscribeNewBlockHeader();
     } catch {

packages/tendermint-rpc/src/tendermint34/adaptor.ts

@@ -23,6 +23,7 @@ export interface Params {
   readonly encodeBlock: (req: requests.BlockRequest) => JsonRpcRequest;
   readonly encodeBlockchain: (req: requests.BlockchainRequest) => JsonRpcRequest;
   readonly encodeBlockResults: (req: requests.BlockResultsRequest) => JsonRpcRequest;
+  readonly encodeBlockSearch: (req: requests.BlockSearchRequest) => JsonRpcRequest;
   readonly encodeBroadcastTx: (req: requests.BroadcastTxRequest) => JsonRpcRequest;
   readonly encodeCommit: (req: requests.CommitRequest) => JsonRpcRequest;
   readonly encodeGenesis: (req: requests.GenesisRequest) => JsonRpcRequest;
@@ -39,6 +40,7 @@ export interface Responses {
   readonly decodeAbciQuery: (response: JsonRpcSuccessResponse) => responses.AbciQueryResponse;
   readonly decodeBlock: (response: JsonRpcSuccessResponse) => responses.BlockResponse;
   readonly decodeBlockResults: (response: JsonRpcSuccessResponse) => responses.BlockResultsResponse;
+  readonly decodeBlockSearch: (response: JsonRpcSuccessResponse) => responses.BlockSearchResponse;
   readonly decodeBlockchain: (response: JsonRpcSuccessResponse) => responses.BlockchainResponse;
   readonly decodeBroadcastTxSync: (response: JsonRpcSuccessResponse) => responses.BroadcastTxSyncResponse;
   readonly decodeBroadcastTxAsync: (response: JsonRpcSuccessResponse) => responses.BroadcastTxAsyncResponse;

packages/tendermint-rpc/src/tendermint34/adaptors/v0-34/requests.ts

@@ -30,6 +30,21 @@ function encodeBlockchainRequestParams(param: requests.BlockchainRequestParams):
   };
 }
 
+interface RpcBlockSearchParams {
+  readonly query: string;
+  readonly page?: string;
+  readonly per_page?: string;
+  readonly order_by?: string;
+}
+function encodeBlockSearchParams(params: requests.BlockSearchParams): RpcBlockSearchParams {
+  return {
+    query: params.query,
+    page: may(Integer.encode, params.page),
+    per_page: may(Integer.encode, params.per_page),
+    order_by: params.order_by,
+  };
+}
+
 interface RpcAbciQueryParams {
   readonly path: string;
   /** hex encoded */
@@ -118,6 +133,10 @@ export class Params {
     return createJsonRpcRequest(req.method, encodeHeightParam(req.params));
   }
 
+  public static encodeBlockSearch(req: requests.BlockSearchRequest): JsonRpcRequest {
+    return createJsonRpcRequest(req.method, encodeBlockSearchParams(req.params));
+  }
+
   public static encodeBroadcastTx(req: requests.BroadcastTxRequest): JsonRpcRequest {
     return createJsonRpcRequest(req.method, encodeBroadcastTxParams(req.params));
   }

packages/tendermint-rpc/src/tendermint34/adaptors/v0-34/responses.ts

@@ -341,15 +341,17 @@ function decodeHeader(data: RpcHeader): responses.Header {
     height: Integer.parse(assertNotEmpty(data.height)),
     time: fromRfc3339WithNanoseconds(assertNotEmpty(data.time)),
 
-    lastBlockId: decodeBlockId(data.last_block_id),
+    // When there is no last block ID (i.e. this block's height is 1), we get an empty structure like this:
+    // { hash: '', parts: { total: 0, hash: '' } }
+    lastBlockId: data.last_block_id.hash ? decodeBlockId(data.last_block_id) : null,
 
-    lastCommitHash: fromHex(assertNotEmpty(data.last_commit_hash)),
+    lastCommitHash: fromHex(assertSet(data.last_commit_hash)),
     dataHash: fromHex(assertSet(data.data_hash)),
 
-    validatorsHash: fromHex(assertNotEmpty(data.validators_hash)),
-    nextValidatorsHash: fromHex(assertNotEmpty(data.next_validators_hash)),
-    consensusHash: fromHex(assertNotEmpty(data.consensus_hash)),
-    appHash: fromHex(assertNotEmpty(data.app_hash)),
+    validatorsHash: fromHex(assertSet(data.validators_hash)),
+    nextValidatorsHash: fromHex(assertSet(data.next_validators_hash)),
+    consensusHash: fromHex(assertSet(data.consensus_hash)),
+    appHash: fromHex(assertSet(data.app_hash)),
     lastResultsHash: fromHex(assertSet(data.last_results_hash)),
 
     evidenceHash: fromHex(assertSet(data.evidence_hash)),
@@ -765,7 +767,9 @@ interface RpcBlock {
 function decodeBlock(data: RpcBlock): responses.Block {
   return {
     header: decodeHeader(assertObject(data.header)),
-    lastCommit: decodeCommit(assertObject(data.last_commit)),
+    // For the block at height 1, last commit is not set. This is represented in an empty object like this:
+    // { height: '0', round: 0, block_id: { hash: '', parts: [Object] }, signatures: [] }
+    lastCommit: data.last_commit.block_id.hash ? decodeCommit(assertObject(data.last_commit)) : null,
     txs: data.data.txs ? assertArray(data.data.txs).map(fromBase64) : [],
     evidence: data.evidence && may(decodeEvidences, data.evidence.evidence),
   };
@@ -783,6 +787,18 @@ function decodeBlockResponse(data: RpcBlockResponse): responses.BlockResponse {
   };
 }
 
+interface RpcBlockSearchResponse {
+  readonly blocks: readonly RpcBlockResponse[];
+  readonly total_count: string;
+}
+
+function decodeBlockSearch(data: RpcBlockSearchResponse): responses.BlockSearchResponse {
+  return {
+    totalCount: Integer.parse(assertNotEmpty(data.total_count)),
+    blocks: assertArray(data.blocks).map(decodeBlockResponse),
+  };
+}
+
 export class Responses {
   public static decodeAbciInfo(response: JsonRpcSuccessResponse): responses.AbciInfoResponse {
     return decodeAbciInfo(assertObject((response.result as AbciInfoResult).response));
@@ -800,6 +816,10 @@ export class Responses {
     return decodeBlockResults(response.result as RpcBlockResultsResponse);
   }
 
+  public static decodeBlockSearch(response: JsonRpcSuccessResponse): responses.BlockSearchResponse {
+    return decodeBlockSearch(response.result as RpcBlockSearchResponse);
+  }
+
   public static decodeBlockchain(response: JsonRpcSuccessResponse): responses.BlockchainResponse {
     return decodeBlockchain(response.result as RpcBlockchainResponse);
   }

packages/tendermint-rpc/src/tendermint34/hasher.ts

@@ -46,6 +46,12 @@ function hashTree(hashes: readonly Uint8Array[]): Uint8Array {
 }
 
 export function hashBlock(header: Header): Uint8Array {
+  if (!header.lastBlockId) {
+    throw new Error(
+      "Hashing a block header with no last block ID (i.e. header at height 1) is not supported. If you need this, contributions are welcome. Please add documentation and test vectors for this case.",
+    );
+  }
+
   const encodedFields: readonly Uint8Array[] = [
     encodeVersion(header.version),
     encodeString(header.chainId),

packages/tendermint-rpc/src/tendermint34/index.ts

@@ -8,6 +8,8 @@ export {
   AbciQueryRequest,
   BlockRequest,
   BlockchainRequest,
+  BlockSearchParams,
+  BlockSearchRequest,
   BlockResultsRequest,
   BroadcastTxRequest,
   BroadcastTxParams,
@@ -38,6 +40,7 @@ export {
   BlockParams,
   BlockResponse,
   BlockResultsResponse,
+  BlockSearchResponse,
   BroadcastTxAsyncResponse,
   BroadcastTxCommitResponse,
   broadcastTxCommitSuccess,

packages/tendermint-rpc/src/tendermint34/requests.ts

@@ -12,6 +12,7 @@ export enum Method {
   /** Get block headers for minHeight <= height <= maxHeight. */
   Blockchain = "blockchain",
   BlockResults = "block_results",
+  BlockSearch = "block_search",
   BroadcastTxAsync = "broadcast_tx_async",
   BroadcastTxSync = "broadcast_tx_sync",
   BroadcastTxCommit = "broadcast_tx_commit",
@@ -30,6 +31,7 @@ export type Request =
   | AbciInfoRequest
   | AbciQueryRequest
   | BlockRequest
+  | BlockSearchRequest
   | BlockchainRequest
   | BlockResultsRequest
   | BroadcastTxRequest
@@ -60,6 +62,7 @@ export interface AbciQueryRequest {
   readonly method: Method.AbciQuery;
   readonly params: AbciQueryParams;
 }
+
 export interface AbciQueryParams {
   readonly path: string;
   readonly data: Uint8Array;
@@ -97,10 +100,23 @@ export interface BlockResultsRequest {
   };
 }
 
+export interface BlockSearchRequest {
+  readonly method: Method.BlockSearch;
+  readonly params: BlockSearchParams;
+}
+
+export interface BlockSearchParams {
+  readonly query: string;
+  readonly page?: number;
+  readonly per_page?: number;
+  readonly order_by?: string;
+}
+
 export interface BroadcastTxRequest {
   readonly method: Method.BroadcastTxAsync | Method.BroadcastTxSync | Method.BroadcastTxCommit;
   readonly params: BroadcastTxParams;
 }
+
 export interface BroadcastTxParams {
   readonly tx: Uint8Array;
 }
@@ -141,6 +157,7 @@ export interface TxRequest {
   readonly method: Method.Tx;
   readonly params: TxParams;
 }
+
 export interface TxParams {
   readonly hash: Uint8Array;
   readonly prove?: boolean;

packages/tendermint-rpc/src/tendermint34/responses.ts

@@ -60,6 +60,11 @@ export interface BlockResultsResponse {
   readonly endBlockEvents: readonly Event[];
 }
 
+export interface BlockSearchResponse {
+  readonly blocks: readonly BlockResponse[];
+  readonly totalCount: number;
+}
+
 export interface BlockchainResponse {
   readonly lastHeight: number;
   readonly blockMetas: readonly BlockMeta[];
@@ -212,7 +217,10 @@ export interface BlockId {
 
 export interface Block {
   readonly header: Header;
-  readonly lastCommit: Commit;
+  /**
+   * For the block at height 1, last commit is not set.
+   */
+  readonly lastCommit: Commit | null;
   readonly txs: readonly Uint8Array[];
   readonly evidence?: readonly Evidence[];
 }
@@ -264,21 +272,39 @@ export interface Header {
   readonly height: number;
   readonly time: ReadonlyDateWithNanoseconds;
 
-  // prev block info
-  readonly lastBlockId: BlockId;
+  /**
+   * Block ID of the previous block. This can be `null` when the currect block is height 1.
+   */
+  readonly lastBlockId: BlockId | null;
 
-  // hashes of block data
+  /**
+   * Hashes of block data.
+   *
+   * This is `sha256("")` for height 1 🤷‍
+   */
   readonly lastCommitHash: Uint8Array;
-  readonly dataHash: Uint8Array; // empty when number of transaction is 0
+  /**
+   * This is `sha256("")` as long as there is no data 🤷‍
+   */
+  readonly dataHash: Uint8Array;
 
   // hashes from the app output from the prev block
   readonly validatorsHash: Uint8Array;
   readonly nextValidatorsHash: Uint8Array;
   readonly consensusHash: Uint8Array;
+  /**
+   * This can be an empty string for height 1 and turn into "0000000000000000" later on 🤷‍
+   */
   readonly appHash: Uint8Array;
+  /**
+   * This is `sha256("")` as long as there is no data 🤷‍
+   */
   readonly lastResultsHash: Uint8Array;
 
   // consensus info
+  /**
+   * This is `sha256("")` as long as there is no data 🤷‍
+   */
   readonly evidenceHash: Uint8Array;
   readonly proposerAddress: Uint8Array;
 }

packages/tendermint-rpc/src/tendermint34/tendermint34client.spec.ts

@@ -204,6 +204,69 @@ function defaultTestSuite(rpcFactory: () => RpcClient, expected: ExpectedValues)
     });
   });
 
+  describe("blockSearch", () => {
+    beforeAll(async () => {
+      if (tendermintEnabled()) {
+        const client = await Tendermint34Client.create(rpcFactory());
+
+        // eslint-disable-next-line no-inner-declarations
+        async function sendTx(): Promise<void> {
+          const tx = buildKvTx(randomString(), randomString());
+
+          const txRes = await client.broadcastTxCommit({ tx: tx });
+          expect(responses.broadcastTxCommitSuccess(txRes)).toEqual(true);
+          expect(txRes.height).toBeTruthy();
+          expect(txRes.hash.length).not.toEqual(0);
+        }
+
+        // send 3 txs
+        await sendTx();
+        await sendTx();
+        await sendTx();
+
+        client.disconnect();
+
+        await tendermintSearchIndexUpdated();
+      }
+    });
+
+    it("can paginate over blockSearch results", async () => {
+      pendingWithoutTendermint();
+      const client = await Tendermint34Client.create(rpcFactory());
+
+      const query = buildQuery({ raw: "block.height >= 1 AND block.height <= 3" });
+
+      // expect one page of results
+      const s1 = await client.blockSearch({ query: query, page: 1, per_page: 2 });
+      expect(s1.totalCount).toEqual(3);
+      expect(s1.blocks.length).toEqual(2);
+
+      // second page
+      const s2 = await client.blockSearch({ query: query, page: 2, per_page: 2 });
+      expect(s2.totalCount).toEqual(3);
+      expect(s2.blocks.length).toEqual(1);
+
+      client.disconnect();
+    });
+
+    it("can get all search results in one call", async () => {
+      pendingWithoutTendermint();
+      const client = await Tendermint34Client.create(rpcFactory());
+
+      const query = buildQuery({ raw: "block.height >= 1 AND block.height <= 3" });
+
+      const sall = await client.blockSearchAll({ query: query, per_page: 2 });
+      expect(sall.totalCount).toEqual(3);
+      expect(sall.blocks.length).toEqual(3);
+      // make sure there are in order from lowest to highest height
+      const [b1, b2, b3] = sall.blocks;
+      expect(b2.block.header.height).toEqual(b1.block.header.height + 1);
+      expect(b3.block.header.height).toEqual(b2.block.header.height + 1);
+
+      client.disconnect();
+    });
+  });
+
   describe("blockchain", () => {
     it("returns latest in descending order by default", async () => {
       pendingWithoutTendermint();

packages/tendermint-rpc/src/tendermint34/tendermint34client.ts

@@ -98,6 +98,53 @@ export class Tendermint34Client {
     return this.doCall(query, this.p.encodeBlockResults, this.r.decodeBlockResults);
   }
 
+  /**
+   * Search for events that are in a block.
+   *
+   * NOTE
+   * This method will error on any node that is running a Tendermint version lower than 0.34.9.
+   *
+   * @see https://docs.tendermint.com/master/rpc/#/Info/block_search
+   */
+  public async blockSearch(params: requests.BlockSearchParams): Promise<responses.BlockSearchResponse> {
+    const query: requests.BlockSearchRequest = { params: params, method: requests.Method.BlockSearch };
+    const resp = await this.doCall(query, this.p.encodeBlockSearch, this.r.decodeBlockSearch);
+    return {
+      ...resp,
+      // make sure we sort by height, as tendermint may be sorting by string value of the height
+      blocks: [...resp.blocks].sort((a, b) => a.block.header.height - b.block.header.height),
+    };
+  }
+
+  // this should paginate through all blockSearch options to ensure it returns all results.
+  // starts with page 1 or whatever was provided (eg. to start on page 7)
+  //
+  // NOTE
+  // This method will error on any node that is running a Tendermint version lower than 0.34.9.
+  public async blockSearchAll(params: requests.BlockSearchParams): Promise<responses.BlockSearchResponse> {
+    let page = params.page || 1;
+    const blocks: responses.BlockResponse[] = [];
+    let done = false;
+
+    while (!done) {
+      const resp = await this.blockSearch({ ...params, page: page });
+      blocks.push(...resp.blocks);
+      if (blocks.length < resp.totalCount) {
+        page++;
+      } else {
+        done = true;
+      }
+    }
+    // make sure we sort by height, as tendermint may be sorting by string value of the height
+    // and the earlier items may be in a higher page than the later items
+    blocks.sort((a, b) => a.block.header.height - b.block.header.height);
+
+    return {
+      totalCount: blocks.length,
+      blocks: blocks,
+    };
+  }
+
   /**
    * Queries block headers filtered by minHeight <= height <= maxHeight.
    *