Search for online accounts (#1646)

* Search for online accounts in `GET /v2/accounts`

* Fix typo

* Fix an issue in the `online-only` filter

Fix an issue in the underlying SQL query for the `online-only`
parameter.

The previous SQL query was returning online accounts with null key
material. This commit filters out those accounts - only accounts
with non-null key material are returned.

* Style fixes

* Add e2e test coverage for `online-only` parameter

* Disable `online-only` parameter by default.

In the endpoint `GET /v2/accounts`, disable the `online-only` parameter
by default.

* Remove unnecesary code

* Increase test coverage for `online-only` param

* Style fixes

* Update disabled parameter list in README.md

* Update param list in `DisablingParametersGuide.md`

* Remove duplicated code

* Add extra parenthesis in SQL expression

* Add missing table alias in SQL query

* Improve readability in e2e test

* Remove unnecessary comments

* Add comments to e2e test

* Update misleading comment

Author: agodnic

README.md

@@ -110,6 +110,7 @@ Below is a snippet of the output from `algorand-indexer api-config`:
     optional:
         - currency-greater-than: disabled
         - currency-less-than: disabled
+        - online-only: disabled
 /v2/assets/{asset-id}/transactions:
     optional:
         - note-prefix: disabled

api/disabled_parameters.go

@@ -317,7 +317,7 @@ func GetDefaultDisabledMapConfigForPostgres() *DisabledMapConfig {
 		rval.addEntry(restPath, http.MethodGet, parameterNames)
 	}
 
-	get("/v2/accounts", []string{"currency-greater-than", "currency-less-than"})
+	get("/v2/accounts", []string{"currency-greater-than", "currency-less-than", "online-only"})
 	get("/v2/accounts/{account-id}/transactions", []string{"note-prefix", "tx-type", "sig-type", "asset-id", "before-time", "after-time", "rekey-to"})
 	get("/v2/assets", []string{"name", "unit"})
 	get("/v2/assets/{asset-id}/balances", []string{"currency-greater-than", "currency-less-than"})

api/error_messages.go

@@ -38,6 +38,8 @@ const (
 	ErrMultipleBoxes                   = "multiple application boxes found for this app id and box name, please contact us, this shouldn't happen"
 	ErrFailedLookingUpBoxes            = "failed while looking up application boxes"
 	errMultiAcctRewind                 = "multiple accounts rewind is not supported by this server"
+	errOnlineOnlyRewind                = "simultaneously rewinding and searching for online accounts is not supported"
+	errOnlineOnlyDeleted               = "simultaneously searching for online and deleted accounts is not supported"
 	errRewindingAccount                = "error while rewinding account"
 	errLookingUpBlockForRound          = "error while looking up block for round"
 	errBlockHeaderSearch               = "error while searching for block headers"

api/generated/common/routes.go

@@ -415,6 +415,14 @@ func (si *ServerImplementation) SearchForAccounts(ctx echo.Context, params gener
 		return badRequest(ctx, errMultiAcctRewind)
 	}
 
+	// Input validations related to the "online-only" parameter
+	if params.Round != nil && boolOrDefault(params.OnlineOnly) {
+		return badRequest(ctx, errOnlineOnlyRewind)
+	}
+	if boolOrDefault(params.OnlineOnly) && boolOrDefault(params.IncludeAll) {
+		return badRequest(ctx, errOnlineOnlyDeleted)
+	}
+
 	var spendingAddrBytes []byte
 	if params.AuthAddr != nil {
 		spendingAddr, err := sdk.DecodeAddress(*params.AuthAddr)
@@ -435,6 +443,7 @@ func (si *ServerImplementation) SearchForAccounts(ctx echo.Context, params gener
 		EqualToAuthAddr:      spendingAddrBytes,
 		IncludeDeleted:       boolOrDefault(params.IncludeAll),
 		MaxResources:         si.opts.MaxAPIResourcesPerAccount,
+		OnlineOnly:           boolOrDefault(params.OnlineOnly),
 	}
 
 	if params.Exclude != nil {

api/generated/common/types.go

@@ -1147,6 +1147,80 @@ func TestVersion(t *testing.T) {
 	require.Equal(t, response.Version, "(unknown version)")
 }
 
+// TestAccountsOnlineOnlyParam exercises the `online-only` parameter in `GET /v2/accounts`.
+func TestAccountsOnlineOnlyParam(t *testing.T) {
+
+	// Spin a fresh database
+	db, shutdownFunc := setupIdb(t, test.MakeGenesis())
+	defer shutdownFunc()
+
+	// Load a block (with a keyreg txn) into the database
+	vb, err := test.ReadValidatedBlockFromFile("test_resources/validated_blocks/KeyregTransactionWithStateProofKeys.vb")
+	require.NoError(t, err)
+	err = db.AddBlock(&vb)
+	require.NoError(t, err)
+	api := &ServerImplementation{db: db}
+
+	e := echo.New()
+	{
+		//////////
+		// When // We query for only online accounts
+		//////////
+		req := httptest.NewRequest(http.MethodGet, "/", nil)
+		rec := httptest.NewRecorder()
+		c := e.NewContext(req, rec)
+		c.SetPath("/v2/accounts")
+		err = api.SearchForAccounts(c, generated.SearchForAccountsParams{OnlineOnly: boolPtr(true)})
+		//////////
+		// Then // Only AccountA should be returned
+		//////////
+		require.NoError(t, err)
+		require.Equal(t, http.StatusOK, rec.Code)
+		var response generated.AccountsResponse
+		data := rec.Body.Bytes()
+		err = json.Decode(data, &response)
+		require.NoError(t, err)
+		require.Equal(t, 1, len(response.Accounts))
+		require.Equal(t, test.AccountA.String(), response.Accounts[0].Address)
+	}
+	{
+		//////////
+		// When // We query for accounts using online-only=false
+		//////////
+		req := httptest.NewRequest(http.MethodGet, "/", nil)
+		rec := httptest.NewRecorder()
+		c := e.NewContext(req, rec)
+		c.SetPath("/v2/accounts")
+		err = api.SearchForAccounts(c, generated.SearchForAccountsParams{OnlineOnly: boolPtr(false)})
+		//////////
+		// Then // All accounts should be returned, regardless of whether their status is online or not
+		//////////
+		require.NoError(t, err)
+		require.Equal(t, http.StatusOK, rec.Code)
+		var response generated.AccountsResponse
+		data := rec.Body.Bytes()
+		err = json.Decode(data, &response)
+		require.NoError(t, err)
+		require.Equal(t, len(test.MakeGenesis().Allocation), len(response.Accounts))
+	}
+	{
+		//////////
+		// When // We query for accounts using both `online-only=true` and `include-all=true` parameters
+		//////////
+		req := httptest.NewRequest(http.MethodGet, "/", nil)
+		rec := httptest.NewRecorder()
+		c := e.NewContext(req, rec)
+		c.SetPath("/v2/accounts")
+		err = api.SearchForAccounts(c, generated.SearchForAccountsParams{OnlineOnly: boolPtr(true), IncludeAll: boolPtr(true)})
+		//////////
+		// Then // The response should be a 404 "bad request"
+		//////////
+		require.NoError(t, err)
+		require.Equal(t, http.StatusBadRequest, rec.Code)
+		require.Contains(t, rec.Body.String(), errOnlineOnlyDeleted)
+	}
+}
+
 func TestAccountClearsNonUTF8(t *testing.T) {
 	db, shutdownFunc := setupIdb(t, test.MakeGenesis())
 	defer shutdownFunc()

api/generated/v2/routes.go

@@ -81,12 +81,14 @@
           {
             "type": "integer",
             "description": "Include results for the specified round. For performance reasons, this parameter may be disabled on some configurations. Using application-id or asset-id filters will return both creator and opt-in accounts. Filtering by include-all will return creator and opt-in accounts for deleted assets and accounts. Non-opt-in managers are not included in the results when asset-id is used.",
-
             "name": "round",
             "in": "query"
           },
           {
             "$ref": "#/parameters/application-id"
+          },
+          {
+            "$ref": "#/parameters/online-only"
           }
         ],
         "responses": {
@@ -2972,6 +2974,12 @@
       "in": "query",
       "x-algorand-format": "base64"
     },
+    "online-only": {
+      "type": "boolean",
+      "description": "When this is set to true, return only accounts whose participation status is currently online.",
+      "name": "online-only",
+      "in": "query"
+    },
     "rekey-to": {
       "type": "boolean",
       "description": "Include results which include the rekey-to field.",

api/generated/v2/types.go

@@ -221,6 +221,14 @@
         },
         "x-algorand-format": "base64"
       },
+      "online-only": {
+        "description": "When this is set to true, return only accounts whose participation status is currently online.",
+        "in": "query",
+        "name": "online-only",
+        "schema": {
+          "type": "boolean"
+        }
+      },
       "proposers": {
         "description": "Accounts marked as proposer in the block header's participation updates. This parameter accepts a comma separated list of addresses.",
         "explode": false,
@@ -2644,6 +2652,14 @@
             "schema": {
               "type": "integer"
             }
+          },
+          {
+            "description": "When this is set to true, return only accounts whose participation status is currently online.",
+            "in": "query",
+            "name": "online-only",
+            "schema": {
+              "type": "boolean"
+            }
           }
         ],
         "responses": {

api/handlers.go

@@ -25,6 +25,7 @@ The configuration file that is used to enable/disable parameters is a YAML file
     optional:
         - currency-greater-than: disabled
         - currency-less-than: disabled
+        - online-only: disabled
 /v2/assets/{asset-id}/transactions:
     optional:
         - note-prefix: disabled

api/handlers_e2e_test.go

@@ -304,6 +304,11 @@ type AccountQueryOptions struct {
 	// return any accounts with this auth addr
 	EqualToAuthAddr []byte
 
+	// OnlineOnly, when set to true, indicates that only accounts that are online should be returned.
+	//
+	// When set to false, this parameter is ignored (i.e. it becomes a no-op).
+	OnlineOnly bool
+
 	// Filter on accounts with current balance greater than x
 	AlgosGreaterThan *uint64
 	// Filter on accounts with current balance less than x.
@@ -326,7 +331,7 @@ type AccountQueryOptions struct {
 	// MaxResources is the maximum combined number of AppParam, AppLocalState, AssetParam, and AssetHolding objects allowed.
 	MaxResources uint64
 
-	// IncludeDeleted indicated whether to include deleted Assets, Applications, etc within the account.
+	// IncludeDeleted indicates whether to include deleted Assets, Applications, etc within the account.
 	IncludeDeleted bool
 
 	Limit uint64

api/indexer.oas2.json

@@ -2031,6 +2031,9 @@ func (db *IndexerDb) buildAccountQuery(opts idb.AccountQueryOptions, countOnly b
 		whereArgs = append(whereArgs, encoding.Base64(opts.EqualToAuthAddr))
 		partNumber++
 	}
+	if opts.OnlineOnly {
+		whereParts = append(whereParts, "((a.account_data->>'onl' IS NOT NULL) AND (a.account_data->>'voteLst' IS NOT NULL) AND ((a.account_data->>'onl') = '1'))")
+	}
 	query = `SELECT a.addr, a.microalgos, a.rewards_total, a.created_at, a.closed_at, a.deleted, a.rewardsbase, a.keytype, a.account_data FROM account a`
 	if opts.HasAssetID != 0 {
 		// inner join requires match, filtering on presence of asset