How can I prevent or abort a transform from running if the input or source dataset is empty?
This code uses a lightweight transform to check if the input DataFrame is empty before writing it to the output location. If the DataFrame is empty, the job is aborted, preventing the writing of empty DataFrames.
Copied!1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17from transforms.api import transform, Input, Output @transform.using( out=Output("/Palantir/output_location/datasets/not_process_empty_files"), source_df=Input("/Palantir/input_location/sometimes_empty"), ) def compute(ctx, source_df, out): source_df = source_df.polars() # Checking if there is at least one row to write if source_df.is_empty(): # No row to write, abort the job ctx.abort_job() else: # Write, hence append. out.write_table(source_df)
code authoring, code repositories, python, abortHow do I create a dataframe inside a transform and write it to the output.
This code defines a function to create an empty DataFrame with a single row containing a dummy key and a timestamp column. It then writes the DataFrame to an output dataset path using a transform decorator.
Copied!1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19from datetime import datetime, timezone import polars as pl from transforms.api import transform, Output # Generation of empty dataframe, that just appends the current timestamp def get_empty_df(): # Create a DataFrame with a single row containing a dummy key # and a 'when' column with the current timestamp return pl.DataFrame({"key": ["dummy_key"], "when": [datetime.now(timezone.utc)]}) @transform.using( out=Output("output_dataset_path") ) def out_1(out): # Create the empty dataframe df = get_empty_df() # Write the dataframe to the output out.write_table(df)
code authoring, code repositories, pythonHow do I perform fuzzy matching of entity names using phonetic codes in PySpark?
This code uses PySpark to clean entity names, generate phonetic codes, and perform fuzzy matching of entity names using the Jaro similarity metric. It is useful for matching similar entity names in two datasets.
Copied!1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62from pyspark.sql import functions as F from pyspark.sql import types as T from transforms.api import transform_df, Input, Output import re import jellyfish def _add_phonetic_codes(df): # Generate phonetic codes for each part of the name df = df.withColumn( "name_part", F.split("cleaned_name", " ") ).withColumn( "name_part", F.explode("name_part") ).withColumn( "phonetic_code", F.soundex("name_part") ).drop("name_part") return df @transform_df( Output(), entities2=Input(), entities1=Input(), ) def compute(entities2, entities1): # Set up UDF for cleaning text def clean_text(text): cleaned_text = re.sub(r" +", " ", re.sub(r"[./-]+", "", text)).lower() return cleaned_text clean_text_udf = F.udf(clean_text, T.StringType()) # Clean entity name entities2 = entities2.withColumn("cleaned_name", clean_text_udf(F.col("name"))) entities1 = entities1.withColumn("cleaned_name", clean_text_udf(F.col("entity_name"))) # Add phonetic codes entities2 = _add_phonetic_codes(entities2) entities1 = _add_phonetic_codes(entities1) # Fuzzy join matched_entities = entities1.join( entities2, on=["phonetic_code"], how="inner" ).select( entities1.cleaned_name.alias("cleaned_name1"), entities1.id.alias("entity_id1"), entities2.cleaned_name.alias("cleaned_name2"), entities2.id.alias("entity_id2") ).drop("phonetic_code") matched_entities = matched_entities.dropDuplicates() # Set up UDF for string comparison @F.udf() def jaro_compare(name1, name2): return jellyfish.jaro_similarity(name1, name2) # Fuzzy matching matched_entities = matched_entities.withColumn( "match_score", jaro_compare("cleaned_name1", "cleaned_name2") ) matched_entities = matched_entities.filter(entities.match_score > 0.75) matched_entities = matched_entities.select("entity_id1", "entity_id2") return matched_entities
pyspark, fuzzy matching, phonetic codes, jaro similarityHow do I load an ORC file using PySpark?
This code reads a raw ORC file from the Hadoop path of an input dataset and writes the resulting spark dataframe to an output.
Copied!1 2 3 4 5 6 7 8 9 10 11from transforms.api import transform, Input, Output @transform( out=Output("output"), raw=Input("input"), ) def compute(ctx, out, raw): hadoop_path = raw.filesystem().hadoop_path df = ctx.spark_session.read.format('orc').load(f'{hadoop_path}/') out.write_dataframe(df)
pyspark, dataframe, orc, hadoopHow do I create a multi-input, multi-output transform in Foundry?
This code demonstrates how to create a lightweight transform that takes multiple input datasets and produces multiple output datasets. It uses the @transform decorator and explicitly names the inputs and outputs. The transform reads the input dataframes, processes them, and writes the results to the specified output datasets.
Copied!1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26from datetime import datetime, timezone import polars as pl from transforms.api import transform, Input, Output, incremental # @incremental decorator or not (compatible with both) # Use @transform instead of @transform_df for explicit control over inputs and outputs. @transform.using( output_dataset_1=Output("3_multi_output_1"), output_dataset_2=Output("3_multi_output_2"), input_dataset_1=Input("fake_dataset"), input_dataset_2=Input("fake_dataset_2") ) def example_transform_multi_inputs_outputs(input_dataset_1, input_dataset_2, output_dataset_1, output_dataset_2): # Inputs can be read input_df_1 = input_dataset_1.polars() input_df_2 = input_dataset_2.polars() # Example processing here processed_at = pl.lit(datetime.now(timezone.utc)).alias("processed_at") only_data_from_1 = input_df_1.with_columns(processed_at) unioned_version = pl.concat([input_df_1, input_df_2.select(input_df_1.columns)], how="vertical_relaxed").with_columns(processed_at) # Instead of returning the modified dataframe, you need to explicitly call "write_table" # on the output of your choice with the dataframe (to write) of your choice. output_dataset_1.write_table(only_data_from_1) output_dataset_2.write_table(unioned_version)
code authoring, code repositories, python