fix dict conversion

method was wrong, and filter on attributes that are in the schema

Author: tobiascadee

tap_looker/client.py

@@ -41,21 +41,19 @@ def schema_filepath(self) -> Path | None:
         """
         return SCHEMAS_DIR / f"{self.name}.json"
 
-    def convert_to_dict(self, obj: object) -> dict:
-        """Convert object to dictionary."""
-        if isinstance(obj, Enum):
-            return obj.value
+    def convert_to_dict(self, obj):
+        if hasattr(obj, "items"):
+            return {key: self.convert_to_dict(value) for key, value in obj.items()}
         if isinstance(obj, list):
             return [self.convert_to_dict(item) for item in obj]
-        if isinstance(obj, dict):
-            return {key: self.convert_to_dict(value) for key, value in obj.items()}
-        if hasattr(obj, "__dict__"):
-            return self.convert_to_dict(obj.__dict__)
+        if isinstance(obj, Enum):
+            return obj.value
         return obj
 
 
 class LookerSystemActivityStream(LookerStream):
     """Stream class for Looker System Activity streams."""
+
     def __init__(self, tap: Tap) -> None:
         """Initialize the stream object."""
         super().__init__(tap)
@@ -84,9 +82,7 @@ def get_records(
         """
         result_count: int = ROW_LIMIT
         result_max_date: datetime = self.get_starting_timestamp(context)
-        fields = [
-            self.replace_prefix(field) for field in list(self.schema["properties"])
-        ]
+        fields = [self.replace_prefix(field) for field in list(self.schema["properties"])]
         while result_count == ROW_LIMIT:
             response = self.sdk.run_inline_query(
                 result_format="json",

tap_looker/streams.py

@@ -44,7 +44,6 @@ def get_child_context(self, record: dict, context: Optional[dict]) -> dict:
             for explore in record.get("explores", [])
             if explore["name"] in self.config.get("filter_explores", [explore["name"]])
         ]
-        self.logger.info(record)
         return {
             "model_name": record["name"],
             "explores": explores,
@@ -65,9 +64,8 @@ def get_records(self, context: Optional[Context]) -> Iterable[Record]:
                 lookml_model_name=context["model_name"],
                 explore_name=explore_name,
             )
-            explore_dict = self.convert_to_dict(explore)
-            for key in ["sets", "fields", "joins", "supported_measure_types"]:
-                explore_dict.pop(key, None)
+            properties = self.schema.get("properties", {}).keys()
+            explore_dict = {key: self.convert_to_dict(value) for key, value in explore.items() if key in properties}
             yield explore_dict
 
 
@@ -85,8 +83,9 @@ def get_records(self, context: Optional[Context]) -> Iterable[Record]:
                 lookml_model_name=context["model_name"],
                 explore_name=explore_name,
             )
-            fields = self.convert_to_dict(explore.fields)
-            for field_type in fields:
-                for item in fields[field_type]:
-                    item["explore_and_name"] = f"{explore_name}.{item['name']}"
-                    yield item
+            properties = self.schema.get("properties", {}).keys()
+            fields = [self.convert_to_dict(field) for field_group in explore.fields.values() for field in field_group]
+            for field in fields:
+                field_dict = {key: value for key, value in field.items() if key in properties}
+                field_dict["explore_and_name"] = f"{explore_name}.{field['name']}"
+                yield field_dict