Warning |
---|
Documentation below does not apply for version >= 3.0.0 |
Use this section if gapi.py is not available in your version
Intended for automatic generation, web UI, and Automation NOT for manual pipeline building
Warning |
---|
Don't use this format |
Intended manual usage in IDE, all settings should have a class for either Dictionary or Pipeline format, so why not use the configuration class directly
No difference with previuos one, regarding functionality, but much more manageable when defining, and modifying
Panel |
---|
Pros
Cons
|
Panel |
---|
Pros
Cons
|
Panel |
---|
Pros
Cons
|
Code Block | ||||||||||
---|---|---|---|---|---|---|---|---|---|---|
| ||||||||||
from utils.constants import DEFAULT_ENGINE_NAME
PIPELINE = {
'enable': True,
'stages': [
{
"type": "HighlightStage",
"enable": True,
"name": "highlight",
"save_to_intermediate": False,
"expand_result": False,
"ui_only": None,
'engine_name': DEFAULT_ENGINE_NAME,
"pre_tags": [
"<strong>"
],
"pos_tags": [
"</strong>"
],
"tags_schema": None,
"force_source": False,
"require_field_match": True,
"number_of_fragments": 100,
"fragment_size": 100,
"order": None,
"fields": [
{
"field": "title",
"fragment_size": None,
"number_of_fragments": 1,
"highlight_query": None,
"type": None,
"force_source": False
},
{
"field": "overview",
"fragment_size": None,
"number_of_fragments": 5,
"highlight_query": None,
"type": None,
"force_source": False
},
{
"field": "url",
"fragment_size": None,
"number_of_fragments": 1,
"highlight_query": None,
"type": None,
"force_source": False
}
]
},
{
"aggs": [
{
"type": "term",
"field": "metadata.genres.name.keyword",
"negated": False,
"must": True,
"id": "genres",
"ui_only": {
"display_name": "Genres",
"hide_count": False,
"hide_if_one": False,
"translate": None,
"max_display": 5,
"mime_icon": False,
"expanded": True,
"enable_negative": True
},
"aggs": [
{
"id": "top_hits",
"type": "top_hits"
},
{
"type": "term",
"field": "metadata.spoken_languages.name.keyword",
"negated": False,
"must": True,
"id": "spoken_languages",
"ui_only": {
"display_name": "Spoken Languages",
"hide_count": False,
"hide_if_one": False,
"translate": None,
"max_display": 5,
"mime_icon": False,
"expanded": False,
"enable_negative": False
},
"aggs": None,
"max_values": None,
"multi_select": False,
"operation": "or",
"order": None,
"exclude_terms": None,
"include_terms": None
}],
"max_values": 100,
"multi_select": True,
"operation": "or",
"order": None,
"exclude_terms": None,
"include_terms": None
},
{
"type": "bool",
"field": "adult",
"negated": False,
"must": True,
"id": "adult",
"ui_only": {
"display_name": "Adult",
"hide_count": False,
"hide_if_one": False,
"translate": None,
"display_True": "True",
"display_False": "False"
},
"aggs": None
},
{
"type": "date_histogram",
"field": "release_date",
"negated": False,
"must": True,
"id": "release_date_h",
"ui_only": {
"display_name": "Release Date",
"hide_if_one": False,
"hide_count": False,
"translate": None,
"level": 0
},
"aggs": None,
"calendar_interval": ["1y", "1q", "1M", "1d"],
"format": ["yyyy", "'Q'q", "MMMM", "E dd"],
"time_zone": ["-06:00"],
"min_doc_count": [100, 1],
"offset": None,
"order": None
},
{
"type": "histogram",
"field": "metadata.budget",
"negated": False,
"must": True,
"id": "budget",
"ui_only": {
"display_name": "Budget",
"hide_if_one": False,
"hide_count": False,
"translate": None,
"level": 0
},
"aggs": None,
"interval": [10000000, 1000000],
"min_doc_count": [100, 10, 1],
"order": None,
"offset": None
},
{
"type": "slider",
"field": "metadata.budget",
"negated": False,
"must": True,
"id": "budget_slider",
"ui_only": {
"display_name": "Budget",
"hide_count": False,
"hide_if_one": False,
"translate": None,
"min": None,
"max": None,
"show_ticks": True
},
"aggs": None
},
{
"type": "date_range",
"field": "release_date",
"negated": False,
"must": True,
"id": "release_date_dr",
"ui_only": {
"display_name": "Release Range",
"hide_count": False,
"hide_if_one": False,
"translate": None
},
"aggs": None,
"format": "MM-yyyy",
"ranges": [
{
"key": "minus_10",
"start": "now-10y",
"start_display": None,
"end": None,
"end_display": None
},
{
"key": None,
"start": "01-1960",
"start_display": None,
"end": "01-1970",
"end_display": None
},
{
"key": None,
"start": None,
"start_display": None,
"end": "now-10M/M",
"end_display": None
}
]
},
{
"type": "range",
"field": "metadata.budget",
"negated": False,
"must": True,
"id": "budget_r",
"ui_only": {
"display_name": "Budget Range",
"hide_count": False,
"hide_if_one": False,
"translate": None
},
"aggs": None,
"ranges": [
{
"key": "< 10,000,000",
"start": None,
"end": "10000000"
},
{
"key": "> 10,000,000",
"start": "10000000",
"end": None
},
{
"key": "50 to 100 millions",
"start": "50000000",
"end": "100000000"
}
],
"multi_select": False
}
],
"type": "DynamicAggStage",
"enable": True,
"name": "aggregations",
"save_to_intermediate": False,
"expand_result": False,
"ui_only": None,
"filters_name": "aggregations_filters",
'engine_name': DEFAULT_ENGINE_NAME,
"search_response": "search",
"ui_aggregations": True
},
{
"type": "FilterStage",
"enable": True,
"name": "filters",
"engine_name": DEFAULT_ENGINE_NAME,
"save_to_intermediate": False,
"expand_result": False,
"filters": [
{
"type": "term",
"field": "adult",
"negated": False,
"must": True,
"value": False,
"multi_select": False
}
]
},
{
'type': 'QueryStage',
'engine_name': DEFAULT_ENGINE_NAME,
'enable': True,
'qpl_enable': True,
'name': 'search',
'index': 'movies',
'fields': ['title', 'overview', 'url', 'status', 'metadata', 'metadata.production_companies',
'metadata.cast',
'metadata.directors'],
'range_fields': ['metadata.release_date'],
'fetch_fields': ['title', 'adult', 'overview', 'url', 'release_date', 'status', 'video', 'metadata',
'metadata.production_companies', 'metadata.cast', 'metadata.directors'],
'implicit_operator': 'or',
'aggregations': 'aggregations',
'aggregations_filters': 'aggregations_filters',
'ui_only': {
'sort': {
'default': {
'field': '_score',
'display_name': 'Score',
'order': 'desc'
},
'options': [
{
'field': '_score',
'display_name': 'Score',
'order': 'desc'
},
{
'field': 'release_date',
'display_name': 'Release Date',
'order': 'asc'
},
{
'field': 'metadata.budget',
'display_name': 'Budget',
'order': 'desc'
}
]
},
'page_size': {
'default': 25,
'options': [25, 50, 100]
}
},
'sort': {
'field': '_score',
'order': 'desc'
},
'page_size_default': 25,
'highlight_stage': 'highlight'
},
{
'type': 'DynamicResultsStage',
'engine_name': DEFAULT_ENGINE_NAME,
'enable': True,
'name': 'hits',
'title': {
'type': 'url',
'field': 'title',
'link': 'url'
},
'sub_title': {
'type': 'text',
'field': 'url',
},
'thumbnail': None,
'non_displayed_fields': [],
'header': [],
'body': [
[{'type': 'bool', 'label': 'Adult', 'field': 'adult'},
{'type': 'number', 'label': 'Popularity', 'field': 'metadata.popularity',
'format': '1.2'}
],
[{'type': 'text', 'field': 'overview'}],
[{'type': 'text', 'label': 'Status', 'field': 'status'},
{'type': 'date', 'label': 'Release Date', 'field': 'release_date',
'format': '%Y-%m-%d'}],
[{'type': 'thumbnails', 'field': 'metadata.cast[*].profile_path_w185',
'alt': 'metadata.cast[*].name', 'detail': 'metadata.cast[*].name',
'full_size': 'metadata.cast[*].profile_path'
}]
],
'metadata': {
'General': {
'fields': [
{
'type': 'chip', 'label': 'Chip', 'field': 'metadata.genres.name',
'unique': True
},
{'type': 'text', 'field': 'title', 'label': 'Title'}
],
'sort': 'asc'
},
},
'search_response': 'search'
},
]
}
|
Code Block | ||||||||||
---|---|---|---|---|---|---|---|---|---|---|
| ||||||||||
from app.pipeline import Pipeline
from app.pipeline.stages import HighlightStage, DynamicAggStage, FilterStage, \
DynamicResultsStage, QueryStage
from app.pipeline.stages.query import QueryStageUIConfig
from framework.aggregations import *
from framework.fields import *
from framework.filters import *
from models.engines import HighlightField, SortConfig, PageSizeConfig, SortEntry
from models.utils import SortOrder
from utils.constants import DEFAULT_ENGINE_NAME
SEARCH_STAGE_NAME = 'search'
########################################################################################################################
# PIPELINE STRUCTURE
########################################################################################################################
PIPELINE = Pipeline(
enable=True,
stages=[
HighlightStage(
enable=True,
name='highlight',
engine_name=DEFAULT_ENGINE_NAME,
pre_tags=['<strong>'],
pos_tags=['</strong>'],
tags_schema=None,
force_source=False,
require_field_match=True,
number_of_fragments=3,
fragment_size=100,
order=None,
fields=[
HighlightField(field='title', number_of_fragments=1),
HighlightField(field='overview', number_of_fragments=5),
HighlightField(field='url', number_of_fragments=1),
]
),
FilterStage(
enable=True,
name='filters',
engine_name=DEFAULT_ENGINE_NAME,
save_to_intermediate=False,
expand_result=False,
filters=[
TermFilter(field='adult', negated=False, must=True, value=False, multi_select=False)
]
),
DynamicAggStage(
enable=True,
name='aggregations',
filters_name='aggregations_filters',
save_to_intermediate=False,
expand_result=False,
engine_name=DEFAULT_ENGINE_NAME,
search_response=SEARCH_STAGE_NAME,
ui_aggregations=True,
# This over here generates the aggregations for ui, if you don't want them, set it to False
aggs=[
TermAgg(
id='genres',
field='metadata.genres.name.keyword',
must=True,
max_values=100,
multi_select=True,
operation='or',
order=None,
exclude_terms=None,
include_terms=None,
ui_only=TermAggUI(display_name='Genres', max_display=5, expanded=True),
aggs=[
TopHitsAgg(
id='top_hits',
type='top_hits'
),
TermAgg(
field='metadata.spoken_languages.name.keyword',
negated=False,
must=True,
id='spoken_languages',
ui_only=TermAggUI(display_name='Spoken Languages', max_display=5, expanded=False),
aggs=None,
max_values=None,
multi_select=False,
operation='or',
order=None,
exclude_terms=None,
include_terms=None
)]
),
BoolAgg(
id='adult',
field='adult',
negated=False,
must=True,
ui_only=BoolAggUI(display_name='Adult', display_True='True', display_False='False')
),
DateHistogramAgg(
field='release_date',
negated=False,
must=True,
id='release_date_h',
ui_only=DateHistogramAggUI(display_name='Release Date'),
calendar_interval=['1y', '1q', '1M', '1d'],
format=['yyyy', '\'Q\'q', 'MMMM', 'E dd'],
time_zone=['-06:00'],
min_doc_count=[100, 1],
offset=None,
order=None
),
HistogramAgg(
id='budget',
field='metadata.budget',
negated=False,
must=True,
interval=[10000000, 1000000],
min_doc_count=[100, 10, 1],
order=None,
offset=None,
ui_only=HistogramAggUI(display_name='Budget')
),
SliderAgg(
id='budget_slider',
field='metadata.budget',
negated=False,
must=True,
ui_only=SliderAggUI(display_name='Budget', translate=None, min=None, max=None, show_ticks=True),
),
DateRangeAgg(
field='release_date',
negated=False,
must=True,
id='release_date_dr',
ui_only=DateRangeAggUI(
display_name='Release Range',
hide_count=False,
translate=None
),
format='MM-yyyy',
ranges=[
DateRange(
key='minus_10',
start='now-10y',
start_display=None,
end=None,
end_display=None
),
DateRange(
key=None,
start='01-1960',
start_display=None,
end='01-1970',
end_display=None
),
DateRange(
key=None,
start=None,
start_display=None,
end='now-10M/M',
end_display=None
)
]
),
RangeAgg(
field='metadata.budget',
negated=False,
must=True,
id='budget_r',
multi_select=False,
ui_only=RangeAggUI(
display_name='Budget Range',
hide_count=False,
translate=None
),
ranges=[
Range(
key='< 10,000,000',
start=None,
end='10000000'
),
Range(
key='> 10,000,000',
start='10000000',
end=None
),
Range(
key='50 to 100 millions',
start='50000000',
end='100000000'
)
],
)
]
),
DynamicResultsStage(
engine_name=DEFAULT_ENGINE_NAME,
enable=True,
name='hits',
title=UrlField(field='title', link='url'),
sub_title=TextField(field='url'),
thumbnail=None,
non_displayed_fields=[],
body=[
[BoolField(label='Adult', field='adult'),
NumberField(label='Popularity', field='metadata.popularity', format='1.2')],
[TextField(field='overview')],
[TextField(label='Status', field='status'),
DateField(label='Release Date', field='release_date', format='%Y-%m-%d')],
[ThumbnailsField(field='metadata.cast[*].profile_path_w185', alt='metadata.cast[*].name',
detail='metadata.cast[*].name', full_size='metadata.cast[*].profile_path')]
],
search_response=SEARCH_STAGE_NAME
),
QueryStage(
engine_name=DEFAULT_ENGINE_NAME,
enable=True,
qpl_enable=True,
name=SEARCH_STAGE_NAME,
index='movies',
wildcard=False,
fields=['title', 'overview', 'url', 'status', 'metadata', 'metadata.production_companies',
'metadata.cast',
'metadata.directors'],
range_fields=['metadata.budget'],
date_fields=['release_date'],
fetch_fields=['title', 'adult', 'overview', 'url', 'release_date', 'status', 'video', 'metadata',
'metadata.production_companies', 'metadata.cast', 'metadata.directors'],
exclude_fields=None,
implicit_operator='or',
page_size_default=25,
sort=SortEntry(
field='_score',
order=SortOrder.DESC
),
aggregations='aggregations',
aggregations_filters='aggregations_filters',
highlight='highlight_stage',
filters='filters',
ui_only=QueryStageUIConfig(
sort=SortConfig(
default=SortEntry(
field='_score',
display_name='Score',
order=SortOrder.DESC
),
options=[
SortEntry(
field='_score',
display_name='Score',
order=SortOrder.DESC
),
SortEntry(
field='release_date',
display_name='Release Date',
order=SortOrder.ASC
),
SortEntry(
field='metadata.budget',
display_name='Budget',
order=SortOrder.DESC
)
]
),
page_size=PageSizeConfig(
default=25,
options=[25, 50, 100]
)
)
)
]
) |
Code Block | ||||||||||
---|---|---|---|---|---|---|---|---|---|---|
| ||||||||||
from app.pipeline import Pipeline
from app.pipeline.stages import HighlightStage, DynamicAggStage, FilterStage, \
DynamicResultsStage, QueryStage
from app.pipeline.stages.query import QueryStageUIConfig
from framework.aggregations import *
from framework.fields import *
from framework.filters import *
from models.engines import HighlightField, SortConfig, PageSizeConfig, SortEntry
from models.utils import SortOrder
from utils.constants import DEFAULT_ENGINE_NAME
########################################################################################################################
# STAGES CONFIGURATION
########################################################################################################################
SEARCH_STAGE_NAME = 'search'
#################
# HIGHLIGHT
#################
_highlight_stage = HighlightStage(
enable=True,
name='highlight',
engine_name=DEFAULT_ENGINE_NAME,
pre_tags=['<strong>'],
pos_tags=['</strong>'],
tags_schema=None,
force_source=False,
require_field_match=True,
number_of_fragments=3,
fragment_size=100,
order=None,
fields=[
HighlightField(field='title', number_of_fragments=1),
HighlightField(field='overview', number_of_fragments=5),
HighlightField(field='url', number_of_fragments=1),
]
)
#################
# AGGREGATIONS
#################
_aggregation_stage = DynamicAggStage(
enable=True,
name='aggregations',
filters_name='aggregations_filters',
save_to_intermediate=False,
expand_result=False,
engine_name=DEFAULT_ENGINE_NAME,
search_response=SEARCH_STAGE_NAME,
ui_aggregations=True, # This over here generates the aggregations for ui, if you don't want them, set it to False
aggs=[
TermAgg(
id='genres',
field='metadata.genres.name.keyword',
must=True,
max_values=100,
multi_select=True,
operation='or',
order=None,
exclude_terms=None,
include_terms=None,
ui_only=TermAggUI(display_name='Genres', max_display=5, expanded=True),
aggs=[
TopHitsAgg(
id='top_hits',
type='top_hits'
),
TermAgg(
field='metadata.spoken_languages.name.keyword',
negated=False,
must=True,
id='spoken_languages',
ui_only=TermAggUI(display_name='Spoken Languages', max_display=5, expanded=False),
aggs=None,
max_values=None,
multi_select=False,
operation='or',
order=None,
exclude_terms=None,
include_terms=None
)]
),
BoolAgg(
id='adult',
field='adult',
negated=False,
must=True,
ui_only=BoolAggUI(display_name='Adult', display_True='True', display_False='False')
),
DateHistogramAgg(
field='release_date',
negated=False,
must=True,
id='release_date_h',
ui_only=DateHistogramAggUI(display_name='Release Date'),
calendar_interval=['1y', '1q', '1M', '1d'],
format=['yyyy', '\'Q\'q', 'MMMM', 'E dd'],
time_zone=['-06:00'],
min_doc_count=[100, 1],
offset=None,
order=None
),
HistogramAgg(
id='budget',
field='metadata.budget',
negated=False,
must=True,
interval=[10000000, 1000000],
min_doc_count=[100, 10, 1],
order=None,
offset=None,
ui_only=HistogramAggUI(display_name='Budget')
),
SliderAgg(
id='budget_slider',
field='metadata.budget',
negated=False,
must=True,
ui_only=SliderAggUI(display_name='Budget', translate=None, min=None, max=None, show_ticks=True),
),
DateRangeAgg(
field='release_date',
negated=False,
must=True,
id='release_date_dr',
ui_only=DateRangeAggUI(
display_name='Release Range',
hide_count=False,
translate=None
),
format='MM-yyyy',
ranges=[
DateRange(
key='minus_10',
start='now-10y',
start_display=None,
end=None,
end_display=None
),
DateRange(
key=None,
start='01-1960',
start_display=None,
end='01-1970',
end_display=None
),
DateRange(
key=None,
start=None,
start_display=None,
end='now-10M/M',
end_display=None
)
]
),
RangeAgg(
field='metadata.budget',
negated=False,
must=True,
id='budget_r',
multi_select=False,
ui_only=RangeAggUI(
display_name='Budget Range',
hide_count=False,
translate=None
),
ranges=[
Range(key='< 10,000,000', start=None, end='10000000'),
Range(key='> 10,000,000', start='10000000', end=None),
Range(key='50 to 100 millions', start='50000000', end='100000000')
],
)
]
)
#################
# FILTER
#################
_filter_stage = FilterStage(
enable=True,
name='filters',
engine_name=DEFAULT_ENGINE_NAME,
save_to_intermediate=False,
expand_result=False,
filters=[
TermFilter(field='adult', negated=False, must=True, value=False, multi_select=False)
]
)
#################
# UI RESULTS
#################
_dynamic_results = DynamicResultsStage(
engine_name=DEFAULT_ENGINE_NAME,
enable=True,
name='hits',
title=UrlField(field='title', link='url'),
sub_title=TextField(field='url'),
thumbnail=None,
non_displayed_fields=[],
body=[
[BoolField(label='Adult', field='adult'),
NumberField(label='Popularity', field='metadata.popularity', format='1.2')],
[TextField(field='overview')],
[TextField(label='Status', field='status'),
DateField(label='Release Date', field='release_date', format='%Y-%m-%d')],
[ThumbnailsField(field='metadata.cast[*].profile_path_w185', alt='metadata.cast[*].name',
detail='metadata.cast[*].name', full_size='metadata.cast[*].profile_path')]
],
search_response=SEARCH_STAGE_NAME
)
#################
# SEARCH
#################
_query_stage = QueryStage(
engine_name=DEFAULT_ENGINE_NAME,
enable=True,
qpl_enable=True,
name=SEARCH_STAGE_NAME,
index='movies',
wildcard=False,
fields=['title', 'overview', 'url', 'status', 'metadata', 'metadata.production_companies',
'metadata.cast',
'metadata.directors'],
range_fields=['metadata.budget'],
date_fields=['release_date'],
fetch_fields=['title', 'adult', 'overview', 'url', 'release_date', 'status', 'video', 'metadata',
'metadata.production_companies', 'metadata.cast', 'metadata.directors'],
exclude_fields=None,
implicit_operator='or',
page_size_default=25,
sort=SortEntry(
field='_score',
order=SortOrder.DESC
),
aggregations=_aggregation_stage.name,
aggregations_filters=_aggregation_stage.filters_name,
highlight=_highlight_stage.name,
filters=_filter_stage.name,
ui_only=QueryStageUIConfig(
sort=SortConfig(
default=SortEntry(
field='_score',
display_name='Score',
order=SortOrder.DESC
),
options=[
SortEntry(
field='_score',
display_name='Score',
order=SortOrder.DESC
),
SortEntry(
field='release_date',
display_name='Release Date',
order=SortOrder.ASC
),
SortEntry(
field='metadata.budget',
display_name='Budget',
order=SortOrder.DESC
)
]
),
page_size=PageSizeConfig(
default=25,
options=[25, 50, 100]
)
)
)
########################################################################################################################
# PIPELINE STRUCTURE
########################################################################################################################
PIPELINE = Pipeline(
enable=True,
stages=[
_highlight_stage,
_filter_stage,
_aggregation_stage,
_dynamic_results,
_query_stage
]
)
|
1) Create the pipeline file
Start by creating a python file for the pipeline configuration in the folder config/pipeline
Tip |
---|
A descriptive name works the best |
On The File
Code Block | ||||
---|---|---|---|---|
| ||||
from app.pipeline import Pipeline from app.pipeline.stages import DemoStage |
Code Block | ||||
---|---|---|---|---|
| ||||
_demo = DemoStage( enable=True, name='demo', save_to_intermediate=False, expand_result=False, ui_only=None, halt_on_exception=False ) |
Code Block | ||||
---|---|---|---|---|
| ||||
PIPELINE = Pipeline( enable=True, stages=[ _demo ] ) |
You can go to /es/docs and execute the GET pipeline endpoint and see if the name appears