BigQuery Schema Generator
GitHub Toggle Dark/Light/Auto mode Toggle Dark/Light/Auto mode Toggle Dark/Light/Auto mode Back to homepage

Bazel

Introduction

In order to use the BQ Schema Generator with Bazel, follow these steps:

  1. Adjust your WORKSPACE file to include your language, protobufs, GRPC, and lastly, the protoc-gen-bq-schema project.
  2. Create a protocol buffer file(s).
    1. Best practices: keep your table definitions (stateful nouns) in a separate file from your transfer objects (stateless) and your service definitions (verbs).
  3. Implement your tests and service end-points (if you have them).
  4. Create your BUILD file in the closet directory to your source code.
  5. Build
  6. Verify your schemas.

Source Code

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
workspace(name = "your_workspace_name")

load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")

# Load the Google API definitions for use with Protobufs
load("//:configs/repository_rules.bzl", "switched_rules_by_language")

switched_rules_by_language(
    name = "com_google_googleapis_imports",
    cc = False,
    csharp = False,
    gapic = False,
    go = True,
    grpc = True,
    java = False,
    nodejs = False,
    php = False,
    python = False,
    ruby = False,
)

###############################################################################
# GO Tool Chain
###############################################################################

http_archive(
    name = "io_bazel_rules_go",
    sha256 = "099a9fb96a376ccbbb7d291ed4ecbdfd42f6bc822ab77ae6f1b5cb9e914e94fa",
    urls = [
        "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.35.0/rules_go-v0.35.0.zip",
        "https://github.com/bazelbuild/rules_go/releases/download/v0.35.0/rules_go-v0.35.0.zip",
    ],
)

http_archive(
    name = "bazel_gazelle",
    sha256 = "501deb3d5695ab658e82f6f6f549ba681ea3ca2a5fb7911154b5aa45596183fa",
    urls = [
        "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.26.0/bazel-gazelle-v0.26.0.tar.gz",
        "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.26.0/bazel-gazelle-v0.26.0.tar.gz",
    ],
)

load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")
load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies")

go_rules_dependencies()

go_register_toolchains(version = "1.19.1")

gazelle_dependencies()

###############################################################################
# Protobuf Tool Chain
###############################################################################

http_archive(
    name = "com_google_protobuf",
    sha256 = "7ba0cb2ecfd9e5d44a6fa9ce05f254b7e5cd70ec89fafba0b07448f3e258310c",
    strip_prefix = "protobuf-21.5",
    urls = [
        "https://github.com/protocolbuffers/protobuf/releases/download/v21.5/protobuf-all-21.5.tar.gz",
    ],
)

http_archive(
    name = "rules_proto",
    sha256 = "e017528fd1c91c5a33f15493e3a398181a9e821a804eb7ff5acdd1d2d6c2b18d",
    strip_prefix = "rules_proto-4.0.0-3.20.0",
    urls = [
        "https://github.com/bazelbuild/rules_proto/archive/refs/tags/4.0.0-3.20.0.tar.gz",
    ],
)

http_archive(
    name = "rules_proto_grpc",
    sha256 = "bbe4db93499f5c9414926e46f9e35016999a4e9f6e3522482d3760dc61011070",
    strip_prefix = "rules_proto_grpc-4.2.0",
    urls = ["https://github.com/rules-proto-grpc/rules_proto_grpc/archive/4.2.0.tar.gz"],
)

load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
load("//:configs/go_deps.bzl", "go_dependencies")

# gazelle:repository_macro configs/go_deps.bzl%go_dependencies
go_dependencies()

protobuf_deps()

http_archive(
    name = "com_github_bazelbuild_buildtools",
    sha256 = "e3bb0dc8b0274ea1aca75f1f8c0c835adbe589708ea89bf698069d0790701ea3",
    strip_prefix = "buildtools-5.1.0",
    urls = [
        "https://github.com/bazelbuild/buildtools/archive/refs/tags/5.1.0.tar.gz",
    ],
)

load("@rules_proto_grpc//:repositories.bzl", "rules_proto_grpc_repos", "rules_proto_grpc_toolchains")

rules_proto_grpc_toolchains()

rules_proto_grpc_repos()

load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")

rules_proto_dependencies()

rules_proto_toolchains()

load("@rules_proto_grpc//go:repositories.bzl", rules_proto_grpc_go_repos = "go_repos")

rules_proto_grpc_go_repos()

load("@rules_proto_grpc//doc:repositories.bzl", rules_proto_grpc_doc_repos = "doc_repos")

rules_proto_grpc_doc_repos()

###############################################################################
# BQ Schema Generator Plugin
###############################################################################
http_archive(
    name = "protoc-gen-bq-schema",
    sha256 = "a9a6d4d84d3053f8a5a4a41be918533fe3fba256ea72a0b9798347b237d55881",
    strip_prefix = "protoc-gen-bq-schema-0.1.0-release",
    url = "https://github.com/rrmcguinness/protoc-gen-bq-schema/archive/refs/tags/0.1.0-release.tar.gz",
)
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
syntax = "proto3";

package example.model;

// Go Lang Options
option go_package = "example/model";

// Java Options
option java_package = "example.model";
option java_multiple_files = true;

// Here for illustration
//import "google/protobuf/timestamp.proto";

import "bq/schema/protos/bq_table.proto";
import "bq/schema/protos/bq_field.proto";

/*
 * ISO 3166-1 Country
 */
message Country {
  option (gen_bq_schema.bigquery_opts).table_name = "country_tbl";
  string id = 1 [
    (gen_bq_schema.bigquery) = {
      require: true
      policy_tags : "private"
    }
  ];
  string name = 2 [
    (gen_bq_schema.bigquery) = {
      require: true
    }
  ];
  string alpha2 = 3 [
    (gen_bq_schema.bigquery) = {
      require: true
    }
  ];
  string alpha3 = 4 [
    (gen_bq_schema.bigquery) = {
      require: true
    }
  ];
  string code = 5;
  string iso2 = 6 [
    (gen_bq_schema.bigquery) = {
      require: true
    }
  ];
  string region = 7;
  string sub_region = 8;
  string intermediate_region = 9;
  string region_code = 10;
  string sub_region_code = 11;
  string intermediate_region_code = 12;
}
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
# For creating a binary proto library
load("@rules_proto//proto:defs.bzl", "proto_library")

# For documenting your protobuf
load("@rules_proto_grpc//doc:defs.bzl", "doc_markdown_compile")

# For compiling the schema
load("//:conf/schema_compile.bzl", "schema_compile")

# Step 1: Define a protocol library, here you can see the dependencies
# required by the proto. The well known types are there for example.
proto_library(
    name = "model",
    srcs = [
        "model.proto",
    ],
    deps = [
        # Add Common Types
        "@com_google_protobuf//:any_proto",
        "@com_google_protobuf//:empty_proto",
        "@com_google_protobuf//:field_mask_proto",
        "@com_google_protobuf//:timestamp_proto",
        # Add the protoc-gen-bq-schema protos
        "@protoc-gen-bq-schema//:bq_field_proto",
        "@protoc-gen-bq-schema//:bq_table_proto",
    ],
    visibility = ["//:__subpackages__"],
)

# Step 2: Compile your schema from the proto library
schema_compile(
    name = "schema",
    extra_protoc_args = [
        "--bq-schema_out=.",
        "--bq-schema_opt=single-message",
    ],
    protos = [
        ":model",
    ],
    visibility = ["//visibility:private"],
)

# Step 3: Use your proto library as a part of your project
go_proto_library(
    name = "my_other_proto_lib",
    importpath = "...",
    proto = "//my_project/api:model",
    visibility = ["//:__subpackages__"],
    deps = [
        "@protoc-gen-bq-schema//:bq_go_proto_lib",
    ],
)
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
[
  {
    "name": "id",
    "type": "STRING",
    "mode": "REQUIRED",
    "policyTags": {
      "names": [
        "private"
      ]
    }
  },
  {
    "name": "name",
    "type": "STRING",
    "mode": "REQUIRED"
  },
  {
    "name": "alpha2",
    "type": "STRING",
    "mode": "REQUIRED"
  },
  {
    "name": "alpha3",
    "type": "STRING",
    "mode": "REQUIRED"
  },
  {
    "name": "code",
    "type": "STRING",
    "mode": "NULLABLE"
  },
  {
    "name": "iso2",
    "type": "STRING",
    "mode": "REQUIRED"
  },
  {
    "name": "region",
    "type": "STRING",
    "mode": "NULLABLE"
  },
  {
    "name": "sub_region",
    "type": "STRING",
    "mode": "NULLABLE"
  },
  {
    "name": "intermediate_region",
    "type": "STRING",
    "mode": "NULLABLE"
  },
  {
    "name": "region_code",
    "type": "STRING",
    "mode": "NULLABLE"
  },
  {
    "name": "sub_region_code",
    "type": "STRING",
    "mode": "NULLABLE"
  },
  {
    "name": "intermediate_region_code",
    "type": "STRING",
    "mode": "NULLABLE"
  }
]