Images tutorial

Note

This is a mirror of images_tutorial.ipynb

To install inferscope pythoh library just run pip install inferscope Then please visit token page https://app.inferscope.tech/token token page, and follow instructions for local token setup. Inferscope library reads token from 3 different places: * ~/.inferscope/token file * INFERSCOPE_TOKEN environment variable * By passing it directly in client constructor

Client(token='IS-Tok-XXXX')

from IPython.display import display, Markdown
import inferscope as infs
import pandas as pd
import numpy as np
import os
from uuid import UUID
from inferscope.models.data_description import (ColumnInformation, DataDescription, DataFormat, DataFormatType,
                                                DataType, ImageSematicProperties, JoinType, SemanticType)
from inferscope.models import ModelInfo, DatasetInfo, Metric, MetricBestValue

Now we have everything needed for this demo imported and ready to run Let's create client and, if needed, setup parent_project_id if specific artifacts/runs location needed

client = infs.Client()
parent_project_id=UUID("bc2b8100-06c9-4526-8b4c-8cc7f239a0d1")

Adding simple run without artifacts

In following cell you can add simple run just to check everything works

def pretty_print_run(run: infs.Run):
    # you can look at raw data
    # print(added_run)
    run_link = f"https://app.inferscope.tech/run/{run.uid}"
    display(Markdown(f'''You can now look at added run in UI by this [link {run_link}]({run_link})'''))

run = infs.Run(
    client=client,
    # You can provide your meaningful run name, or just leave it empty
    # name=None,
    name="Example run without artifacts",
    metrics=[
        Metric(name='avg_time', value=np.random.randn() * 3 + 7, best_value="min"),
        Metric(name='quality_score', value=np.random.randn() * 10, slice="person", best_value="max"),
        Metric(name='quality_score', value=np.random.randn() * 10, slice="animals", best_value="max"),

    ],
    model=ModelInfo(
        name="some model",
        version="0.0.1",
        description="Some model produced or used in run. "
        "We plan to add more sophisticated schema (produced model(s), consumed model(s)) later, if needed - contact the team."
    ),
    dataset=DatasetInfo(
        name="some dataset",
        version="any string can be a version",
        description="Some dataset produced or used in this run. "
        "We plan to add more sophisticated schema (produced model(s), consumed model(s)) later, if needed - contact the team."
    ),
    parent_project_uid=parent_project_id,
    description='''# Markdown description
Example run description. Can contain all markdown elements such as [links](https://inferscope.tech) and other.
    '''
)

# Metric can be logged with this function
run.log_global_metric(
    name='quality_score',
    value=np.random.randn() * 10,
    best_value="max"
)
run.commit()
pretty_print_run(run)

Now adding our first run with artifact - externally store dataframe with links to publicly available content

We prepared some publicly available datasets in our s3 bucket by this links: https://storage.googleapis.com/inferscope_example_data/left_img_dataset.tsv https://storage.googleapis.com/inferscope_example_data/right_img_dataset.tsv

Those datasets contain 4 columns - ids, prompts, generation_time, img_urls

First, we need to define data schema for this dataframe

data_description = DataDescription(
    data_format=DataFormat(
        dsv_delimiter='\t', # separator, can be any separator, behavior like in pandas.read_csv
        data_format=DataFormatType.DSV # we support DSV, delimiter separated format or Json in each line (aka JSONEachRow)
    ),
    columns=[
        ColumnInformation(name="ids", type="integer"),
        ColumnInformation(name="generation_time", type="float", semantic="metric"),
        ColumnInformation(name="img_urls", type="url", semantic="image"),
        ColumnInformation(name="prompts", type="string"),
    ],
)

When working with external links, it's possible to skip ArtifactPack creation and pass list of ExternalLinkArtifact directly in Run constructor.

run = infs.Run(
    client=client,
    artifacts=[
        infs.ExternalLinkArtifact(
            data_description=data_description,
            path="artifact",
            uri=f"https://storage.googleapis.com/inferscope_example_data/left_img_dataset.tsv",
            semantic="dataframe",
        )
    ],
    dataset=DatasetInfo(name=f"example dataset for ext_links", version="0.0.1"),
    metrics=[
        Metric(name="some_metric_name", value=0.8),
    ],
    name=f'Example run with ext link artifacts'
)
run.commit()
pretty_print_run(run)
print("Please open 'Dataframe view' tap in ui to see the data")

Uploading artifacts to inferscope internal storage

We also support storing datarfames and content in our storage. We have infs.StoredArtifactHelper for uploading data and getting special links iss:// that our UI & Python library can recognize and show/download their content. We generated some images with bing img gen and saved them in generated_imgs folder

!ls generated_imgs

Let's upload one of those images and use it in dataframe with it's prompt (extracted from filename)

local_img_path = "generated_imgs/adventurous-fox-knight-in-armor--a-dashing-cartoon.jpg"
prompt = "adventurous fox knight in armor a dashing cartoon"
run = infs.Run(
    name="run with single image",
    parent_project_id=parent_project_id,
)

simpe_df = pd.DataFrame([{
    "ids": 1,
    "img_urls": run.log_image(
        name="adventurous-fox-knight-in-armor--a-dashing-cartoon.jpg",
        local_path=local_img_path,
    ), # we just put our iss:// url so artifact or diff viewer can display it properly
    "prompts": prompt,
    "generation_time": 1.381
}])
# upload_artifact can work with binary and regual strings, so let's try it
run.add_artifact(
    path="dataframe",
    blob=simpe_df.to_csv(sep='\t',index=None),
    data_description=data_description
)
run.commit()
pretty_print_run(run)

Now to the most important example - multi lines example, ready for diffing!

def gen(right=False):
    run = infs.Run(
        metrics=[
            Metric(name='avg_time', value=np.random.randn() * 3 + 7, best_value="min"),
            Metric(name='quality_score', value=np.random.randn() * 10, best_value="max"),
            Metric(name='quality_score', value=np.random.randn() * 10, slice="person", best_value="max"),
            Metric(name='quality_score', value=np.random.randn() * 10, slice="animals", best_value="max"),

        ],
        parent_project_uid=parent_project_id,
        description='''#Markdown description
Example run description. Can contain all markdown elements such as [links](https://inferscope.tech), lists
        '''
    )
    names = sorted(list(filter(lambda x: x.endswith('jpg') and (('_right' in x) == right),os.listdir('generated_imgs'))))
    d = pd.DataFrame()
    d['ids'] = [i for i in range(len(names))]
    d['prompts'] = [' '.join(name[:-4].split('-')) for name in names]
    d['generation_time'] = np.random.randn(len(names)) * 3 + 7
    d['img_urls'] = [
        run.log_image(
            name,
            local_path=f'generated_imgs/{name}',
            ) for name in names
    ]
    run.add_artifact(
        blob=d.to_csv(sep='\t',index=None),
        path='dataframe',
        data_description=data_description,
    )


    return client.add(run)
right_run = gen(True)
pretty_print_run(right_run)
left_run = gen(False)
pretty_print_run(left_run)
direct_diff_url = f'https://app.inferscope.tech/diff_view?run_uids={left_run.uid},{right_run.uid}'
display(Markdown(f'''
Now you can visit UI, or just this direct url [{direct_diff_url}]({direct_diff_url})
'''
))