8000 GCP / Google workspace by mjsprengers · Pull Request #111 · Blazebit/blaze-query · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
49 commits
Select commit Hold shift + click to select a range
16b3312
feat: get signInActivity (only allowed if tenant has premium p1 or p2…
mjsprengers Oct 30, 2024
d4e5d3b
feat: add signInActivity to response if tenant has premium p1 or p2 l…
mjsprengers Oct 30, 2024
bb9af2d
feat: handle the case where Intune is not used / not licensed, return…
mjsprengers Oct 31, 2024
71ab332
feat: add organization datafetcher
mjsprengers Nov 4, 2024
f9d5f61
Merge pull request #1 from mjsprengers/graphservice-datafetchers
mjsprengers Nov 4, 2024
1ef3874
Revert "feat: add organization datafetcher"
mjsprengers Nov 7, 2024
7f04c50
Revert "feat: handle the case where Intune is not used / not licensed…
mjsprengers Nov 7, 2024
42cf40e
Revert "feat: add signInActivity to response if tenant has premium p1…
mjsprengers Nov 7, 2024
94c616a
Revert "feat: get signInActivity (only allowed if tenant has premium …
mjsprengers Nov 7, 2024
fe66bff
Merge branch 'Blazebit:main' into main
mjsprengers Nov 11, 2024
847c6f5
Merge branch 'Blazebit:main' into main
mjsprengers Nov 14, 2024
80d4248
Merge branch 'Blazebit:main' into main
mjsprengers Nov 27, 2024
8a2d48d
Merge branch 'Blazebit:main' into main
mjsprengers Dec 23, 2024
61ab53b
Merge branch 'Blazebit:main' into main
mjsprengers Jan 28, 2025
809f0a4
Merge branch 'Blazebit:main' into main
mjsprengers Feb 7, 2025
8ff1310
Merge branch 'Blazebit:main' into main
mjsprengers Feb 12, 2025
ed13417
Merge branch 'Blazebit:main' into main
mjsprengers Feb 27, 2025
4f1aaba
Merge branch 'Blazebit:main' into main
mjsprengers Mar 5, 2025
b578b8b
Merge branch 'Blazebit:main' into main
mjsprengers Jun 27, 2025
6e3dbb4
Merge branch 'Blazebit:main' into main
mjsprengers Sep 2, 2025
63982ac
Merge branch 'Blazebit:main' into main
mjsprengers Dec 7, 2025
d04a40a
Merge branch 'Blazebit:main' into main
mjsprengers Jan 20, 2026
30a5ba4
feat: add workspace tests
mjsprengers Jan 20, 2026
8e1a994
comment + spotless
mjsprengers Jan 20, 2026
1e45804
add google setup
mjsprengers Jan 26, 2026
c1e8692
add projects that are not necessarily part of an organisation
mjsprengers Feb 5, 2026
b8f91a0
add LazyStringArrayList filter to google context
mjsprengers Feb 5, 2026
ec6ec2c
enable gcp
mjsprengers Feb 5, 2026
6ecf03a
filter protocolstringlist to avoid NoSuchElementException
mjsprengers Feb 9, 2026
bbf5d99
introduce GcpWrapper
mjsprengers Feb 9, 2026
5c49eb9
update EnumerableTableScan to use FakeComparableList for result assig…
mjsprengers Feb 9, 2026
1c497e2
Add payload wrappers for google workspace and compute
mjsprengers Feb 10, 2026
9f6ef87
remove specific code
mjsprengers Feb 10, 2026
cc4a636
Convert protobuf Duration and Timestamp to java.time types in GCP con…
mjsprengers Feb 16, 2026
4150ed6
Add ConvertingFieldAccessor support to EnumerableTableScan. Only acti…
mjsprengers Feb 16, 2026
dbe5e0f
Use ConvertingFieldAccessor for protobuf type resolution
mjsprengers Feb 16, 2026
6ba4a11
add gcp instance and bucket tests
mjsprengers Feb 16, 2026
b151ca5
Merge branch 'Blazebit:main' into main
mjsprengers Feb 18, 2026
4d61d3b
Merge branch 'Blazebit:main' into main
mjsprengers Mar 1, 2026
2df08a2
Merge branch 'refs/heads/main' into gcp_fetchers
mjsprengers Mar 1, 2026
137e97b
fix tests
mjsprengers Mar 3, 2026
5e0c458
Merge branch 'Blazebit:main' into main
mjsprengers Mar 11, 2026
2ef8a8b
Merge branch 'main' into gcp_fetchers
mjsprengers Mar 11, 2026
ccd5a23
feat: add google workspace endpoint verification
mjsprengers Mar 12, 2026
d4c104f
comment out google connectors
mjsprengers Mar 12, 2026
ffd43d9
Fix pr comments
mjsprengers Mar 12, 2026
711c158
spotless
mjsprengers Mar 12, 2026
b5df66e
fix spring boot 4 migration
mjsprengers Mar 13, 2026
32d40bd
Fix endpoint verification wrappers to use GoogleWrapper pattern
maxhov Mar 13, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10000
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
*/
package com.blazebit.query.connector.base;

import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;

import com.blazebit.query.spi.DataFormatFieldAccessor;
Expand Down Expand Up @@ -46,7 +47,7 @@ public Object get(Object o) {
}
return converterMethod.invoke( null, value );
}
catch (Exception e) {
catch (InvocationTargetException | IllegalAccessException e) {
throw new RuntimeException( e );
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,6 @@
import com.google.cloud.asset.v1.Asset;
import com.google.cloud.asset.v1.AssetServiceClient;
import com.google.cloud.asset.v1.AssetServiceSettings;
import com.google.cloud.resourcemanager.v3.Folder;
import com.google.cloud.resourcemanager.v3.Organization;
import com.google.cloud.resourcemanager.v3.Project;

import java.io.Serializable;
import java.util.ArrayList;
Expand All @@ -25,49 +22,49 @@
* @author Christian Beikov
* @since 1.0.0
*/
public class AssetDataFetcher implements DataFetcher<Asset>, Serializable {
public class AssetDataFetcher implements DataFetcher<GcpAsset>, Serializable {

public static final AssetDataFetcher INSTANCE = new AssetDataFetcher();

private AssetDataFetcher() {
}

@Override
public List<Asset> fetch(DataFetchContext context) {
public List<GcpAsset> fetch(DataFetchContext context) {
try {
List<CredentialsProvider> credentialsProviders = GcpConnectorConfig.GCP_CREDENTIALS_PROVIDER.getAll( context );
List<Asset> list = new ArrayList<>();
List<? extends Organization> organizations = context.getSession().getOrFetch( Organization.class );
List<? extends Folder> folders = context.getSession().getOrFetch( Folder.class );
List<? extends Project> projects = context.getSession().getOrFetch( Project.class );
List<GcpAsset> list = new ArrayList<>();
List<? extends GcpOrganization> organizations = context.getSession().getOrFetch( GcpOrganization.class );
List<? extends GcpFolder> folders = context.getSession().getOrFetch( GcpFolder.class );
List<? extends GcpProject> projects = context.getSession().getOrFetch( GcpProject.class );

for ( CredentialsProvider credentialsProvider : credentialsProviders ) {
final AssetServiceSettings settings = AssetServiceSettings.newBuilder()
.setCredentialsProvider(credentialsProvider)
.build();
try (AssetServiceClient client = AssetServiceClient.create( settings )) {
for ( Organization organization : organizations ) {
for ( GcpOrganization organization : organizations ) {
final AssetServiceClient.ListAssetsPagedResponse response = client.listAssets(
organization.getName()
organization.getPayload().getName()
);
for ( Asset instance : response.iterateAll() ) {
list.add( instance );
list.add( new GcpAsset( instance.getName(), instance ) );
}
}
for ( Folder folder : folders ) {
for ( GcpFolder folder : folders ) {
final AssetServiceClient.ListAssetsPagedResponse response = client.listAssets(
folder.getName()
folder.getPayload().getName()
);
for ( Asset instance : response.iterateAll() ) {
list.add( instance );
list.add( new GcpAsset( instance.getName(), instance ) );
}
}
for ( Project project : projects ) {
for ( GcpProject project : projects ) {
final AssetServiceClient.ListAssetsPagedResponse response = client.listAssets(
project.getName()
project.getPayload().getName()
);
for ( Asset instance : response.iterateAll() ) {
list.add( instance );
list.add( new GcpAsset( instance.getName(), instance ) );
}
}
}
Expand All @@ -81,6 +78,6 @@ public List<Asset> fetch(DataFetchContext context) {

@Override
public DataFormat getDataFormat() {
return DataFormats.beansConvention( Asset.class, GcpConventionContext.INSTANCE );
return DataFormats.beansConvention( GcpAsset.class, GcpConventionContext.INSTANCE );
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
import com.google.cloud.resourcemanager.v3.Folder;
import com.google.cloud.resourcemanager.v3.FoldersClient;
import com.google.cloud.resourcemanager.v3.FoldersSettings;
import com.google.cloud.resourcemanager.v3.Organization;

import java.io.Serializable;
import java.util.ArrayList;
Expand All @@ -23,29 +22,29 @@
* @author Christian Beikov
* @since 1.0.0
*/
public class FolderDataFetcher implements DataFetcher<Folder>, Serializable {
public class FolderDataFetcher implements DataFetcher<GcpFolder>, Serializable {

public static final FolderDataFetcher INSTANCE = new FolderDataFetcher();

private FolderDataFetcher() {
}

@Override
public List<Folder> fetch(DataFetchContext context) {
public List<GcpFolder> fetch(DataFetchContext context) {
try {
List<CredentialsProvider> credentialsProviders = GcpConnectorConfig.GCP_CREDENTIALS_PROVIDER.getAll( context );
List<Folder> list = new ArrayList<>();
List<? extends Organization> organizations = context.getSession().getOrFetch( Organization.class );
List<GcpFolder> list = new ArrayList<>();
List<? extends GcpOrganization> organizations = context.getSession().getOrFetch( GcpOrganization.class );

for ( CredentialsProvider credentialsProvider : credentialsProviders ) {
for ( Organization organization : organizations ) {
for ( GcpOrganization organization : organizations ) {
final FoldersSettings settings = FoldersSettings.newBuilder()
.setCredentialsProvider( credentialsProvider )
.build();
try (FoldersClient client = FoldersClient.create( settings )) {
final FoldersClient.ListFoldersPagedResponse response = client.listFolders( organization.getName() );
final FoldersClient.ListFoldersPagedResponse response = client.listFolders( organization.getPayload().getName() );
for ( Folder instance : response.iterateAll() ) {
list.add( instance );
list.add( new GcpFolder( instance.getName(), instance ) );
}
}
}
Expand All @@ -59,6 +58,6 @@ public List<Folder> fetch(DataFetchContext context) {

@Override
public DataFormat getDataFormat() {
return DataFormats.beansConvention( Folder.class, GcpConventionContext.INSTANCE );
return DataFormats.beansConvention( GcpFolder.class, GcpConventionContext.INSTANCE );
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
/*
* SPDX-License-Identifier: Apache-2.0
* Copyright Blazebit
*/
package com.blazebit.query.connector.gcp.base;

import com.google.cloud.asset.v1.Asset;

public class GcpAsset extends GcpWrapper<Asset> {
public GcpAsset(String resourceId, Asset asset) {
super(resourceId, asset);
}

@Override
public Asset getPayload() {
return super.getPayload();
}
}
< 10BC8 td id="diff-71f8ea5616ec55b1b66e2b1d67f0c80e8238e93ca2e49bc7d379c4c9098371e4R46" data-line-number="46" class="blob-num blob-num-context js-linkable-line-number js-blob-rnum">
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,14 @@
package com.blazebit.query.connector.gcp.base;

import com.blazebit.query.connector.base.ConventionContext;
import com.blazebit.query.connector.base.ConvertingFieldAccessor;
import com.blazebit.query.spi.DataFormatFieldAccessor;
import com.google.protobuf.ByteString;

import java.lang.reflect.Member;
import java.lang.reflect.Method;
import java.time.Duration;
import java.time.Instant;

/**
* A method filter to exclude internal and cyclic methods from the GCP models.
Expand All @@ -20,25 +24,58 @@ public class GcpConventionContext implements ConventionContext {

public static final ConventionContext INSTANCE = new GcpConventionContext();

private static final Method CONVERT_DURATION;
private static final Method CONVERT_TIMESTAMP;

static {
try {
CONVERT_DURATION = GcpTypeConverters.class.getMethod( "convertDuration", com.google.protobuf.Duration.class );
CONVERT_TIMESTAMP = GcpTypeConverters.class.getMethod( "convertTimestamp", com.google.protobuf.Timestamp.class );
}
catch (NoSuchMethodException e) {
throw new RuntimeException( e );
}
}

private GcpConventionContext() {
}

@Override
public ConventionContext getSubFilter(Class<?> concreteClass, Member member) {
final Method method = (Method) member;
if ( method.getReturnType() == ByteString.class
|| com.google.protobuf.ProtocolStringList.class.isAssignableFrom(method.getReturnType())
|| method.getName().endsWith( "OrBuilder" )
|| method.getName().endsWith( "OrBuilderList" )
|| method.getDeclaringClass().getPackageName().startsWith( "com.google.protobuf" ) ) {
return null;
}
switch ( method.getName() ) {
case "getDefaultInstanceForType":
case "getParserForType":
case "getSerializedSize":
return null;
default:
return this;
return switch ( method.getName() ) {
case "getDefaultInstanceForType", "getParserForType", "getSerializedSize" -> null;
default -> this;
};
}

@Override
public Class<?> resolveType(Class<?> typeClass) {
if ( typeClass == com.google.protobuf.Duration.class ) {
return Duration.class;
}
if ( typeClass == com.google.protobuf.Timestamp.class ) {
return Instant.class;
}
return typeClass;
}

@Override
public DataFormatFieldAccessor createConvertingAccessor(
DataFormatFieldAccessor accessor, Class<?> sourceType, Class<?> targetType) {
if ( sourceType == com.google.protobuf.Duration.class ) {
return new ConvertingFieldAccessor( accessor, CONVERT_DURATION );
}
if ( sourceType == com.google.protobuf.Timestamp.class ) {
return new ConvertingFieldAccessor( accessor, CONVERT_TIMESTAMP );
}
return accessor;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
/*
* SPDX-License-Identifier: Apache-2.0
* Copyright Blazebit
*/
package com.blazebit.query.connector.gcp.base;

import com.google.cloud.resourcemanager.v3.Folder;

public class GcpFolder extends GcpWrapper<Folder> {
public GcpFolder(String resourceId, Folder folder) {
super(resourceId, folder);
}

@Override
public Folder getPayload() {
return super.getPayload();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
/*
* SPDX-License-Identifier: Apache-2.0
* Copyright Blazebit
*/
package com.blazebit.query.connector.gcp.base;

import com.google.cloud.resourcemanager.v3.Organization;

public class GcpOrganization extends GcpWrapper<Organization> {
public GcpOrganization(String resourceId, Organization organization) {
super(resourceId, organization);
}

@Override
public Organization getPayload() {
return super.getPayload();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
/*
* SPDX-License-Identifier: Apache-2.0
* Copyright Blazebit
*/
package com.blazebit.query.connector.gcp.base;

import com.google.cloud.resourcemanager.v3.Project;

public class GcpProject extends GcpWrapper<Project> {
public GcpProject(String resourceId, Project project) {
super(resourceId, project);
}

@Override
public Project getPayload() {
return super.getPayload();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
/*
* SPDX-License-Identifier: Apache-2.0
* Copyright Blazebit
*/
package com.blazebit.query.connector.gcp.base;

import java.time.Duration;
import java.time.Instant;

/**
* Static conversion methods for protobuf types to java.time types.
* Default/unset protobuf instances (0s, 0ns) are converted to {@code null}.
*
* @author Christian Beikov
* @since 1.0.0
*/
public final class GcpTypeConverters {

private GcpTypeConverters() {
}

/**
* Converts a protobuf {@link com.google.protobuf.Duration} to a {@link Duration}.
*
* @param d The protobuf duration
* @return The java.time duration, or {@code null} if the protobuf duration is the default instance
*/
public static Duration convertDuration(com.google.protobuf.Duration d) {
if ( d == null || d.equals( com.google.protobuf.Duration.getDefaultInstance() ) ) {
return null;
}
return Duration.ofSeconds( d.getSeconds(), d.getNanos() );
}

/**
* Converts a protobuf {@link com.google.protobuf.Timestamp} to an {@link Instant}.
*
* @param t The protobuf timestamp
* @return The java.time instant, or {@code null} if the protobuf timestamp is the default instance
*/
public static Instant convertTimestamp(com.google.protobuf.Timestamp t) {
if ( t == null || t.equals( com.google.protobuf.Timestamp.getDefaultInstance() ) ) {
return null;
}
return Instant.ofEpochSecond( t.getSeconds(), t.getNanos() );
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
/*
* SPDX-License-Identifier: Apache-2.0
* Copyright Blazebit
*/
package com.blazebit.query.connector.gcp.base;

public abstract class GcpWrapper<T> {

private final String resourceId;
private final T payload;

public GcpWrapper(String resourceId, T payload) {
this.resourceId = resourceId;
this.payload = payload;
}

public String getResourceId() {
return resourceId;
}

public T getPayload() {
return payload;
}
}
Loading
Loading
0