Examples

These protocol buffer files demonstrate how to use the tags to generate different output specs.

Common Protobuf

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11


syntax = "proto3";

import "google/protobuf/timestamp.proto";

option java_package = "somewhere";

message Shared {
    // Required: The message origination domain
    string data_descriptor = 1;
}

Foo (Proto 3)

Protobuf

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51


syntax = "proto3";

package foo;

import "bq_table.proto";
import "bq_field.proto";

import "google/protobuf/wrappers.proto";
import "google/protobuf/timestamp.proto";

import "common.proto";

message Bar {
  option (gen_bq_schema.bigquery_opts).table_name = "bar_proto3_table";

  message Nested {
    repeated int32 a = 1;
  }

  // Description of field a -- this is an int32
  int32 a = 1;

  // Nested b structure
  Nested b = 2;

  // Repeated c string
  repeated string c = 3;

  bool d = 4 [(gen_bq_schema.bigquery).ignore = true];

  // TIMESTAMP (uint64 in proto) - required in BigQuery
  uint64 e = 5 [
    (gen_bq_schema.bigquery) = {
      require: true
      type_override: 'TIMESTAMP'
    }
  ];

  google.protobuf.Int32Value wkt1 = 11;
  google.protobuf.Timestamp wkt2 = 12;

  // Sub-message within the file
  Baz sub_baz = 13;

  // Shared data defined in common.proto
  Shared share_data = 14;
}

message Baz {
  int32 a = 1;
}

Schema

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70


[
 {
  "name": "a",
  "type": "INTEGER",
  "mode": "NULLABLE",
  "description": "Description of field a -- this is an int32"
 },
 {
  "name": "b",
  "type": "RECORD",
  "mode": "NULLABLE",
  "description": "Nested b structure",
  "fields": [
   {
    "name": "a",
    "type": "INTEGER",
    "mode": "REPEATED"
   }
  ]
 },
 {
  "name": "c",
  "type": "STRING",
  "mode": "REPEATED",
  "description": "Repeated c string"
 },
 {
  "name": "e",
  "type": "TIMESTAMP",
  "mode": "REQUIRED",
  "description": "TIMESTAMP (uint64 in proto) - required in BigQuery"
 },
 {
  "name": "wkt1",
  "type": "INTEGER",
  "mode": "NULLABLE"
 },
 {
  "name": "wkt2",
  "type": "TIMESTAMP",
  "mode": "NULLABLE"
 },
 {
  "name": "sub_baz",
  "type": "RECORD",
  "mode": "NULLABLE",
  "description": "Sub-message within the file",
  "fields": [
   {
    "name": "a",
    "type": "INTEGER",
    "mode": "NULLABLE"
   }
  ]
 },
 {
  "name": "share_data",
  "type": "RECORD",
  "mode": "NULLABLE",
  "description": "Shared data defined in common.proto",
  "fields": [
   {
    "name": "data_descriptor",
    "type": "STRING",
    "mode": "NULLABLE",
    "description": "Required: The message origination domain"
   }
  ]
 }
]

Foo (Proto 2)

proto | schema

Protobuf

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41


syntax = "proto2";
package foo;
import "bq_table.proto";
import "bq_field.proto";

import "google/protobuf/wrappers.proto";
import "google/protobuf/timestamp.proto";

message Bar {
  option (gen_bq_schema.bigquery_opts).table_name = "bar_table";

  message Nested {
    repeated int32 a = 1;
  }

  // Description of field a -- this is an int32
  required int32 a = 1;

  // Nested b structure
  optional Nested b = 2;

  // Repeated c string
  repeated string c = 3;

  optional bool d = 4 [(gen_bq_schema.bigquery).ignore = true];

  // TIMESTAMP (uint64 in proto) - required in BigQuery
  optional uint64 e = 5 [
    (gen_bq_schema.bigquery) = {
      require: true
      type_override: 'TIMESTAMP'
    }
  ];

  optional google.protobuf.Int32Value wkt1 = 11;
  optional google.protobuf.Timestamp wkt2 = 12;
}

message Baz {
  required int32 a = 1;
}

Schema

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43


[
 {
  "name": "a",
  "type": "INTEGER",
  "mode": "REQUIRED",
  "description": "Description of field a -- this is an int32"
 },
 {
  "name": "b",
  "type": "RECORD",
  "mode": "NULLABLE",
  "description": "Nested b structure",
  "fields": [
   {
    "name": "a",
    "type": "INTEGER",
    "mode": "REPEATED"
   }
  ]
 },
 {
  "name": "c",
  "type": "STRING",
  "mode": "REPEATED",
  "description": "Repeated c string"
 },
 {
  "name": "e",
  "type": "TIMESTAMP",
  "mode": "REQUIRED",
  "description": "TIMESTAMP (uint64 in proto) - required in BigQuery"
 },
 {
  "name": "wkt1",
  "type": "INTEGER",
  "mode": "NULLABLE"
 },
 {
  "name": "wkt2",
  "type": "TIMESTAMP",
  "mode": "NULLABLE"
 }
]

Single Message

Protobuf

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27


syntax = "proto2";
package foo;

import "google/protobuf/wrappers.proto";
import "google/protobuf/timestamp.proto";

message Bar {
  message Nested {
    repeated int32 a = 1;
  }

  // Description of field a -- this is an int32
  required int32 a = 1;

  // Nested b structure
  optional Nested b = 2;

  // Repeated c string
  repeated string c = 3;

  optional google.protobuf.Int32Value wkt1 = 11;
  optional google.protobuf.Timestamp wkt2 = 12;
}

message Baz {
  required int32 a = 1;
}

Schema

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37


[
 {
  "name": "a",
  "type": "INTEGER",
  "mode": "REQUIRED",
  "description": "Description of field a -- this is an int32"
 },
 {
  "name": "b",
  "type": "RECORD",
  "mode": "NULLABLE",
  "description": "Nested b structure",
  "fields": [
   {
    "name": "a",
    "type": "INTEGER",
    "mode": "REPEATED"
   }
  ]
 },
 {
  "name": "c",
  "type": "STRING",
  "mode": "REPEATED",
  "description": "Repeated c string"
 },
 {
  "name": "wkt1",
  "type": "INTEGER",
  "mode": "NULLABLE"
 },
 {
  "name": "wkt2",
  "type": "TIMESTAMP",
  "mode": "NULLABLE"
 }
]

Test Table

Protobuf

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33


syntax = "proto3";
package foo;
import "bq_table.proto";
import "bq_field.proto";

message TestTable{
    option (gen_bq_schema.bigquery_opts).table_name = "test_table";

    int32 a = 1 [
        (gen_bq_schema.bigquery) = {
          require: true
          policy_tags : "private"
        }
      ];
    
    string b = 2 [(gen_bq_schema.bigquery).policy_tags="public"];

    message Nested {
        int32 a = 1 [(gen_bq_schema.bigquery) = {
            require: true
            policy_tags : "private"
            }
        ];

        string b = 2;
    }

    repeated Nested nested = 3 [(gen_bq_schema.bigquery).require = true];

    message EmptyMessage {}

    repeated EmptyMessage hasMessage = 4;
}

Schema

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44


[
 {
  "name": "a",
  "type": "INTEGER",
  "mode": "REQUIRED",
  "policyTags": {
   "names": [
    "private"
   ]
  }
 },
 {
  "name": "b",
  "type": "STRING",
  "mode": "NULLABLE",
  "policyTags": {
   "names": [
    "public
   ]
  }
 },
 {
  "name": "nested",
  "type": "RECORD",
  "mode": "REQUIRED",
  "fields": [
   {
    "name": "a",
    "type": "INTEGER",
    "mode": "REQUIRED",
    "policyTags": {
     "names": [
      "private"
     ]
    }
   },
   {
    "name": "b",
    "type": "STRING",
    "mode": "NULLABLE"
   }
  ]
 }
]