Dataset retention service (gRPC)#

Manages retention policies and exemptions for datasets.

Endpoints#

Delete dataset retention exemption#

gRPC /nominal.storage.v1.DatasetRetentionService/DeleteDatasetRetentionExemption

Delete a retention exemption for a dataset.

Request: DeleteDatasetRetentionExemptionRequest

dataset_rid string required
source_id string required

Response: DeleteDatasetRetentionExemptionResponse

A message with no fields.

Request

Python

import grpc
from nominal.protos.storage.v1 import dataset_retention_pb2, dataset_retention_pb2_grpc

channel = grpc.secure_channel("api.gov.nominal.io:443", grpc.ssl_channel_credentials())
dataset_retention_service = dataset_retention_pb2_grpc.DatasetRetentionServiceStub(channel)
auth = [("authorization", "Bearer <api key>")]

request = dataset_retention_pb2.DeleteDatasetRetentionExemptionRequest(
    dataset_rid="<dataset rid>",
    source_id="<source id>",
)
response = dataset_retention_service.DeleteDatasetRetentionExemption(request, metadata=auth)
print(response)

package main

import (
	"context"
	"crypto/tls"
	"fmt"
	"log"

	storagev1 "github.com/nominal-io/nominal-api-protos-go/nominal/protos/storage/v1"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/metadata"
)

func main() {
	conn, err := grpc.NewClient("api.gov.nominal.io:443",
		grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{})))
	if err != nil {
		log.Fatal(err)
	}
	defer conn.Close()
	datasetRetentionService := storagev1.NewDatasetRetentionServiceClient(conn)
	ctx := metadata.AppendToOutgoingContext(context.Background(), "authorization", "Bearer <api key>")

	request := &storagev1.DeleteDatasetRetentionExemptionRequest{
		DatasetRid: "<dataset rid>",
		SourceId:   "<source id>",
	}
	response, err := datasetRetentionService.DeleteDatasetRetentionExemption(ctx, request)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(response)
}

Response

{}

Get dataset retention watermark#

gRPC /nominal.storage.v1.DatasetRetentionService/GetDatasetRetentionWatermark

Get the retention watermark for a dataset.

Request: GetDatasetRetentionWatermarkRequest

dataset_rid string required

Response: GetDatasetRetentionWatermarkResponse

watermark Timestamp

Fields of Timestamp

seconds integer (int64)
nanos integer (int64)

Request

Python

import grpc
from nominal.protos.storage.v1 import dataset_retention_pb2, dataset_retention_pb2_grpc

channel = grpc.secure_channel("api.gov.nominal.io:443", grpc.ssl_channel_credentials())
dataset_retention_service = dataset_retention_pb2_grpc.DatasetRetentionServiceStub(channel)
auth = [("authorization", "Bearer <api key>")]

request = dataset_retention_pb2.GetDatasetRetentionWatermarkRequest(
    dataset_rid="<dataset rid>",
)
response = dataset_retention_service.GetDatasetRetentionWatermark(request, metadata=auth)
print(response)

package main

import (
	"context"
	"crypto/tls"
	"fmt"
	"log"

	storagev1 "github.com/nominal-io/nominal-api-protos-go/nominal/protos/storage/v1"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/metadata"
)

func main() {
	conn, err := grpc.NewClient("api.gov.nominal.io:443",
		grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{})))
	if err != nil {
		log.Fatal(err)
	}
	defer conn.Close()
	datasetRetentionService := storagev1.NewDatasetRetentionServiceClient(conn)
	ctx := metadata.AppendToOutgoingContext(context.Background(), "authorization", "Bearer <api key>")

	request := &storagev1.GetDatasetRetentionWatermarkRequest{
		DatasetRid: "<dataset rid>",
	}
	response, err := datasetRetentionService.GetDatasetRetentionWatermark(ctx, request)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(response)
}

Response

{
  "watermark": {
    "seconds": 1,
    "nanos": 1
  }
}

List dataset retention exemptions#

gRPC /nominal.storage.v1.DatasetRetentionService/ListDatasetRetentionExemptions

List retention exemptions for a dataset, filtered to those overlapping the given time range.

Request: ListDatasetRetentionExemptionsRequest

dataset_rid string required

bounds Range required

Filter exemptions that overlap with this time range.

Fields of Range

start Timestamp

Fields of Timestamp

seconds integer (int64)
nanos integer (int64)

end Timestamp

page_size integer

From 1 to 1,000.

next_page_token string

Response: ListDatasetRetentionExemptionsResponse

exemptions array of DatasetRetentionExemption

Fields of DatasetRetentionExemption

dataset_rid string

The dataset this exemption applies to.

source_id string

Identifier for the source of this exemption (e.g. run RID).

bounds Range

Inclusive time range to exempt from retention.

Fields of Range

start Timestamp

Fields of Timestamp

seconds integer (int64)
nanos integer (int64)

end Timestamp

description string

At most 1,024 characters.

next_page_token string

Request

Python

import grpc
from nominal.protos.storage.v1 import dataset_retention_pb2, dataset_retention_pb2_grpc

channel = grpc.secure_channel("api.gov.nominal.io:443", grpc.ssl_channel_credentials())
dataset_retention_service = dataset_retention_pb2_grpc.DatasetRetentionServiceStub(channel)
auth = [("authorization", "Bearer <api key>")]

request = dataset_retention_pb2.ListDatasetRetentionExemptionsRequest(
    dataset_rid="<dataset rid>",
    bounds={"start": {"seconds": 1, "nanos": 1}, "end": {"seconds": 1, "nanos": 1}},
)
response = dataset_retention_service.ListDatasetRetentionExemptions(request, metadata=auth)
print(response)

package main

import (
	"context"
	"crypto/tls"
	"fmt"
	"log"

	storagev1 "github.com/nominal-io/nominal-api-protos-go/nominal/protos/storage/v1"
	typestime "github.com/nominal-io/nominal-api-protos-go/nominal/protos/types/time"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/metadata"
	"google.golang.org/protobuf/proto"
)

func main() {
	conn, err := grpc.NewClient("api.gov.nominal.io:443",
		grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{})))
	if err != nil {
		log.Fatal(err)
	}
	defer conn.Close()
	datasetRetentionService := storagev1.NewDatasetRetentionServiceClient(conn)
	ctx := metadata.AppendToOutgoingContext(context.Background(), "authorization", "Bearer <api key>")

	request := &storagev1.ListDatasetRetentionExemptionsRequest{
		DatasetRid: "<dataset rid>",
		Bounds: &typestime.Range{
			Start: &typestime.Timestamp{
				Seconds: proto.Int64(1),
				Nanos:   proto.Int64(1),
			},
			End: &typestime.Timestamp{
				Seconds: proto.Int64(1),
				Nanos:   proto.Int64(1),
			},
		},
	}
	response, err := datasetRetentionService.ListDatasetRetentionExemptions(ctx, request)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(response)
}

Response

{
  "exemptions": [
    {
      "dataset_rid": "<dataset rid>",
      "source_id": "<source id>",
      "bounds": {
        "start": {
          "seconds": 1,
          "nanos": 1
        },
        "end": {
          "seconds": 1,
          "nanos": 1
        }
      },
      "description": "<description>"
    }
  ],
  "next_page_token": "<next page token>"
}

Update dataset retention watermark#

gRPC /nominal.storage.v1.DatasetRetentionService/UpdateDatasetRetentionWatermark

Update the retention watermark for a dataset.

Request: UpdateDatasetRetentionWatermarkRequest

dataset_rid string required

watermark Timestamp required

Fields of Timestamp

seconds integer (int64)
nanos integer (int64)

Response: UpdateDatasetRetentionWatermarkResponse

watermark Timestamp

Fields of Timestamp

seconds integer (int64)
nanos integer (int64)

Request

Python

import grpc
from nominal.protos.storage.v1 import dataset_retention_pb2, dataset_retention_pb2_grpc

channel = grpc.secure_channel("api.gov.nominal.io:443", grpc.ssl_channel_credentials())
dataset_retention_service = dataset_retention_pb2_grpc.DatasetRetentionServiceStub(channel)
auth = [("authorization", "Bearer <api key>")]

request = dataset_retention_pb2.UpdateDatasetRetentionWatermarkRequest(
    dataset_rid="<dataset rid>",
    watermark={"seconds": 1, "nanos": 1},
)
response = dataset_retention_service.UpdateDatasetRetentionWatermark(request, metadata=auth)
print(response)

package main

import (
	"context"
	"crypto/tls"
	"fmt"
	"log"

	storagev1 "github.com/nominal-io/nominal-api-protos-go/nominal/protos/storage/v1"
	typestime "github.com/nominal-io/nominal-api-protos-go/nominal/protos/types/time"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/metadata"
	"google.golang.org/protobuf/proto"
)

func main() {
	conn, err := grpc.NewClient("api.gov.nominal.io:443",
		grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{})))
	if err != nil {
		log.Fatal(err)
	}
	defer conn.Close()
	datasetRetentionService := storagev1.NewDatasetRetentionServiceClient(conn)
	ctx := metadata.AppendToOutgoingContext(context.Background(), "authorization", "Bearer <api key>")

	request := &storagev1.UpdateDatasetRetentionWatermarkRequest{
		DatasetRid: "<dataset rid>",
		Watermark: &typestime.Timestamp{
			Seconds: proto.Int64(1),
			Nanos:   proto.Int64(1),
		},
	}
	response, err := datasetRetentionService.UpdateDatasetRetentionWatermark(ctx, request)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(response)
}

Response

{
  "watermark": {
    "seconds": 1,
    "nanos": 1
  }
}

Update retention policy#

gRPC /nominal.storage.v1.DatasetRetentionService/UpdateRetentionPolicy

Update or remove the retention policy for a dataset. When a policy is set, data older than the policy’s expiry will be deleted.

Request: UpdateRetentionPolicyRequest

dataset_rid string required

retention_policy RetentionPolicy

If absent, the retention policy is removed (reverts to default/keep-forever).

Fields of RetentionPolicy

one of keep_forever, keep_until_expiry required

keep_forever

keep_forever KeepForever

keep_until_expiry

keep_until_expiry KeepUntilExpiry

Fields of KeepUntilExpiry

expiry_days integer: Greater than 0.

Response: UpdateRetentionPolicyResponse

A message with no fields.

Request

Python

import grpc
from nominal.protos.storage.v1 import dataset_retention_pb2, dataset_retention_pb2_grpc

channel = grpc.secure_channel("api.gov.nominal.io:443", grpc.ssl_channel_credentials())
dataset_retention_service = dataset_retention_pb2_grpc.DatasetRetentionServiceStub(channel)
auth = [("authorization", "Bearer <api key>")]

request = dataset_retention_pb2.UpdateRetentionPolicyRequest(
    dataset_rid="<dataset rid>",
)
response = dataset_retention_service.UpdateRetentionPolicy(request, metadata=auth)
print(response)

package main

import (
	"context"
	"crypto/tls"
	"fmt"
	"log"

	storagev1 "github.com/nominal-io/nominal-api-protos-go/nominal/protos/storage/v1"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/metadata"
)

func main() {
	conn, err := grpc.NewClient("api.gov.nominal.io:443",
		grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{})))
	if err != nil {
		log.Fatal(err)
	}
	defer conn.Close()
	datasetRetentionService := storagev1.NewDatasetRetentionServiceClient(conn)
	ctx := metadata.AppendToOutgoingContext(context.Background(), "authorization", "Bearer <api key>")

	request := &storagev1.UpdateRetentionPolicyRequest{
		DatasetRid: "<dataset rid>",
	}
	response, err := datasetRetentionService.UpdateRetentionPolicy(ctx, request)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(response)
}

Response

{}

Upsert dataset retention exemption#

gRPC /nominal.storage.v1.DatasetRetentionService/UpsertDatasetRetentionExemption

Create or update a retention exemption for a dataset. The exemption is keyed by (dataset_rid, source_id) — upserting with the same key updates the time range.

Request: UpsertDatasetRetentionExemptionRequest

dataset_rid string required

source_id string required

bounds Range required

Inclusive time range to exempt from retention.

Fields of Range

start Timestamp

Fields of Timestamp

seconds integer (int64)
nanos integer (int64)

end Timestamp

description string

At most 1,024 characters.

Response: UpsertDatasetRetentionExemptionResponse

exemption DatasetRetentionExemption

Fields of DatasetRetentionExemption

dataset_rid string

The dataset this exemption applies to.

source_id string

Identifier for the source of this exemption (e.g. run RID).

bounds Range

Inclusive time range to exempt from retention.

Fields of Range

start Timestamp

Fields of Timestamp

seconds integer (int64)
nanos integer (int64)

end Timestamp

description string

At most 1,024 characters.

Request

Python

import grpc
from nominal.protos.storage.v1 import dataset_retention_pb2, dataset_retention_pb2_grpc

channel = grpc.secure_channel("api.gov.nominal.io:443", grpc.ssl_channel_credentials())
dataset_retention_service = dataset_retention_pb2_grpc.DatasetRetentionServiceStub(channel)
auth = [("authorization", "Bearer <api key>")]

request = dataset_retention_pb2.UpsertDatasetRetentionExemptionRequest(
    dataset_rid="<dataset rid>",
    source_id="<source id>",
    bounds={"start": {"seconds": 1, "nanos": 1}, "end": {"seconds": 1, "nanos": 1}},
)
response = dataset_retention_service.UpsertDatasetRetentionExemption(request, metadata=auth)
print(response)

package main

import (
	"context"
	"crypto/tls"
	"fmt"
	"log"

	storagev1 "github.com/nominal-io/nominal-api-protos-go/nominal/protos/storage/v1"
	typestime "github.com/nominal-io/nominal-api-protos-go/nominal/protos/types/time"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/metadata"
	"google.golang.org/protobuf/proto"
)

func main() {
	conn, err := grpc.NewClient("api.gov.nominal.io:443",
		grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{})))
	if err != nil {
		log.Fatal(err)
	}
	defer conn.Close()
	datasetRetentionService := storagev1.NewDatasetRetentionServiceClient(conn)
	ctx := metadata.AppendToOutgoingContext(context.Background(), "authorization", "Bearer <api key>")

	request := &storagev1.UpsertDatasetRetentionExemptionRequest{
		DatasetRid: "<dataset rid>",
		SourceId:   "<source id>",
		Bounds: &typestime.Range{
			Start: &typestime.Timestamp{
				Seconds: proto.Int64(1),
				Nanos:   proto.Int64(1),
			},
			End: &typestime.Timestamp{
				Seconds: proto.Int64(1),
				Nanos:   proto.Int64(1),
			},
		},
	}
	response, err := datasetRetentionService.UpsertDatasetRetentionExemption(ctx, request)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(response)
}

Response

{
  "exemption": {
    "dataset_rid": "<dataset rid>",
    "source_id": "<source id>",
    "bounds": {
      "start": {
        "seconds": 1,
        "nanos": 1
      },
      "end": {
        "seconds": 1,
        "nanos": 1
      }
    },
    "description": "<description>"
  }
}

Types#

DatasetRetentionExemption#

A retention exemption for a dataset, representing a time range that should be preserved regardless of the dataset’s retention policy.

dataset_rid string required: The dataset this exemption applies to.
source_id string required: Identifier for the source of this exemption (e.g. run RID).
bounds Range required: Inclusive time range to exempt from retention.
description string: At most 1,024 characters.

KeepForever#

A message with no fields.

KeepUntilExpiry#

expiry_days integer: Greater than 0.

Range#

Deprecated for new APIs. Define an API-owned range with the endpoint semantics it needs instead.

start Timestamp
end Timestamp

RetentionPolicy#

Set exactly one of the fields below.

one of keep_forever, keep_until_expiry required

keep_forever

keep_forever KeepForever

keep_until_expiry

keep_until_expiry KeepUntilExpiry

Timestamp#

Deprecated for new APIs. Use google.protobuf.Timestamp for absolute instants.

This legacy nanosecond-precision timestamp is represented by an epoch time in seconds and a nanosecond offset. The nanosecond offset is from the start of the epoch second, so must be less than 1 billion.

seconds integer (int64)
nanos integer (int64)

Page design based on Scalar, MIT License.