Power Query ignore missing column - excel

I have made a custom function to transform files in a power query:
let
Source = (#"Sample File Parameter1" as binary) => let
Source = Excel.Workbook(#"Sample File Parameter1", null, true),
#"Sjælland m# kl# 1" = Source{[Name="Sjælland m# kl# 0"]}[Data],
#"Removed Top Rows" = Table.Skip(#"Sjælland m# kl# 1",6),
#"Promoted Headers" = Table.PromoteHeaders(#"Removed Top Rows", [PromoteAllScalars=true]),
#"Changed Type" = Table.TransformColumnTypes(#"Promoted Headers",{{"Column1", type text}, {"LAB Nr", type text}, {"Column3", type text}, {"PrøveId", type text}, {"Dybde", type text}, {"Column6", type text}, {"Enhed", type text}, {"Klassificering", type text}, {"Bly (Pb)", type text}, {"Column10", type text}, {"Cadmium (Cd)", type text}, {"Chrom Total (Cr total)", type text}, {"Kobber (Cu)", type number}, {"Nikkel (Ni)", Int64.Type}, {"Zink (Zn)", Int64.Type}, {"Benzen", type text}, {"BTEX total", type text}, {"Benz(a)pyren", type text}, {"Dibenz(a,h)antracen", type text}, {"PAH total", type number}, {"Flygtige (Benzin) (C6-C10)", type text}, {"Let olie (C10-C15)", type text}, {"Let olie (C15-C20)", type text}, {"Tung olie (C20-C35)", type text}, {"Olie Total (C6-C35)", type text}}),
#"Removed Other Columns" = Table.SelectColumns(#"Changed Type",{"PrøveId", "Dybde", "Bly (Pb)", "Cadmium (Cd)", "Chrom Total (Cr total)", "Kobber (Cu)", "Nikkel (Ni)", "Zink (Zn)", "Benzen", "BTEX total", "Benz(a)pyren", "Dibenz(a,h)antracen", "PAH total", "Flygtige (Benzin) (C6-C10)", "Let olie (C10-C15)", "Let olie (C15-C20)", "Tung olie (C20-C35)", "Olie Total (C6-C35)"}),
#"Filtered Rows" = Table.SelectRows(#"Removed Other Columns", each ([#"Olie Total (C6-C35)"] <> null)),
#"Added Custom" = Table.AddColumn(#"Filtered Rows", "Top", each Number.ToText(Number.FromText(Text.Start([Dybde],Text.PositionOf([Dybde],"-"))),"0.0")),
#"Added Custom1" = Table.AddColumn(#"Added Custom", "Key", each [PrøveId]&"-"&[Top]),
#"Removed Other Columns1" = Table.SelectColumns(#"Added Custom1",{"Bly (Pb)", "Cadmium (Cd)", "Chrom Total (Cr total)", "Kobber (Cu)", "Nikkel (Ni)", "Zink (Zn)", "Benzen", "BTEX total", "Benz(a)pyren", "Dibenz(a,h)antracen", "PAH total", "Flygtige (Benzin) (C6-C10)", "Let olie (C10-C15)", "Let olie (C15-C20)", "Tung olie (C20-C35)", "Olie Total (C6-C35)", "Key"}),
#"Unpivoted Only Selected Columns" = Table.Unpivot(#"Removed Other Columns1", {"Bly (Pb)", "Cadmium (Cd)", "Chrom Total (Cr total)", "Kobber (Cu)", "Nikkel (Ni)", "Zink (Zn)", "Benzen", "BTEX total", "Benz(a)pyren", "Dibenz(a,h)antracen", "PAH total", "Flygtige (Benzin) (C6-C10)", "Let olie (C10-C15)", "Let olie (C15-C20)", "Tung olie (C20-C35)", "Olie Total (C6-C35)"}, "Attribute", "Value" )
in
#"Unpivoted Only Selected Columns"
in
Source
I then run the following query to do this to all files in the folder:
let
Source = Folder.Files(ResultsTable),
#"Filtered Hidden Files1" = Table.SelectRows(Source, each [Attributes]?[Hidden]? <> true),
#"Invoke Custom Function1" = Table.AddColumn(#"Filtered Hidden Files1", "Transform File from Analyseresultater", each #"Transform File from Analyseresultater"([Content])),
#"Renamed Columns1" = Table.RenameColumns(#"Invoke Custom Function1", {"Name", "Source.Name"}),
#"Removed Other Columns1" = Table.SelectColumns(#"Renamed Columns1", {"Source.Name", "Transform File from Analyseresultater"}),
#"Expanded Table Column1" = Table.ExpandTableColumn(#"Removed Other Columns1", "Transform File from Analyseresultater", Table.ColumnNames(#"Transform File from Analyseresultater"(#"Sample File"))),
#"Removed Other Columns" = Table.SelectColumns(#"Expanded Table Column1",{"Key", "Attribute", "Value"})
in
#"Removed Other Columns"
My problem then is that the colums "Benzen" and "BTEX Total" are not always present, so I get an error:
Expression.Error: The column 'Benzen' of the table wasn't found.
Details:
Benzen
I have tried using MissingField.Ignore og MissingField.UseNull without luck, so now I turn to StackOverflow for help, can someone help me get past this error :) Hope it's possible

For Table.SelectColumns you may use MissingField.Ignore parameter, as you mentioned. The error returned because of Table.TransformColumnTypes function. The easiest solution - removing {"Benzen", type text}, {"BTEX total", type text}, from it (I guess, type text is not essential for other transformations).

Related

Check if Table is empty and end query if true else continue query

I inherited a fairly complex excel workbook and the following query is throwing an error for the users because there are no null values in the #"Filtered Rows" step, it just returns an empty table. So, I'm trying to figure out if there's a way to have the query stop there if the table is empty but if it's not continue on. Thanks for any advice on how to go about this.
I'm trying to add the #"Check Empty Table" step.
Source = Excel.CurrentWorkbook(){[Name="SAPCrosstab5"]}[Content],
#"Promoted Headers" = Table.PromoteHeaders(Source, [PromoteAllScalars=true]),
#"Changed Type" = Table.TransformColumnTypes(#"Promoted Headers",{{"Planning Reference", type text}, {"Column2", type text}, {"Project Definition", type text}, {"Column4", type text}, {"Distribution Channel", type text}, {"Distribution Channel_1", type text}, {"Contract Fee Type", type text}, {"Comp VS Sole Source", type text}, {"Outside/Assist (Rplan)", type text}, {"Booking Disposition (Rplan)", type text}, {"", type text}, {"Sold To Party", type text}}),
#"Merged Queries" = Table.NestedJoin(#"Changed Type",{"Planning Reference"},#"AOP losses filter",{"Planning Reference"},"AOP losses filter",JoinKind.LeftOuter),
#"Expanded AOP losses filter" = Table.ExpandTableColumn(#"Merged Queries", "AOP losses filter", {"Planning Reference"}, {"Planning Reference.1"}),
#"Filtered Rows" = Table.SelectRows(#"Expanded AOP losses filter", each ([Planning Reference.1] = null) and ([#"Booking Disposition (Rplan)"] = "Non-firm / Opportunity")),
#"Check Empty Table" = if Table.IsEmpty(#"Filtered Rows") then Stop Query else #"Filtered Rows",
#"Removed Columns" = Table.RemoveColumns(#"Filtered Rows",{"Project Definition", "Column4", "Distribution Channel", "Planning Reference.1"}),
#"Pivoted Column" = Table.Pivot(#"Removed Columns", List.Distinct(#"Removed Columns"[#""]), "", "_2", List.Sum),
#"Removed Columns1" = Table.RemoveColumns(#"Pivoted Column",{"BBN", "Total Sales", "Total Profit", "Total Bookings"}),
#"Replaced Value" = Table.ReplaceValue(#"Removed Columns1","Non-firm / Opportunity","Inactive/Delete",Replacer.ReplaceText,{"Booking Disposition (Rplan)"}),
#"Renamed Columns" = Table.RenameColumns(#"Replaced Value",{{"Distribution Channel_1", "Distribution Channel"}})
in
#"Renamed Columns"
It is actually trivial when you understand the let ... in ... can be nested.
If I am making no syntax mistake, this should be it:
let
Source = Excel.CurrentWorkbook(){[Name="SAPCrosstab5"]}[Content],
#"Promoted Headers" = Table.PromoteHeaders(Source, [PromoteAllScalars=true]),
#"Changed Type" = Table.TransformColumnTypes(#"Promoted Headers",{{"Planning Reference", type text}, {"Column2", type text}, {"Project Definition", type text}, {"Column4", type text}, {"Distribution Channel", type text}, {"Distribution Channel_1", type text}, {"Contract Fee Type", type text}, {"Comp VS Sole Source", type text}, {"Outside/Assist (Rplan)", type text}, {"Booking Disposition (Rplan)", type text}, {"", type text}, {"Sold To Party", type text}}),
#"Merged Queries" = Table.NestedJoin(#"Changed Type",{"Planning Reference"},#"AOP losses filter",{"Planning Reference"},"AOP losses filter",JoinKind.LeftOuter),
#"Expanded AOP losses filter" = Table.ExpandTableColumn(#"Merged Queries", "AOP losses filter", {"Planning Reference"}, {"Planning Reference.1"}),
#"Filtered Rows" = Table.SelectRows(#"Expanded AOP losses filter", each ([Planning Reference.1] = null) and ([#"Booking Disposition (Rplan)"] = "Non-firm / Opportunity")),
result = if Table.IsEmpty(#"Filtered Rows") then #"Filtered Rows" else (
let
#"Removed Columns" = Table.RemoveColumns(#"Filtered Rows",{"Project Definition", "Column4", "Distribution Channel", "Planning Reference.1"}),
#"Pivoted Column" = Table.Pivot(#"Removed Columns", List.Distinct(#"Removed Columns"[#""]), "", "_2", List.Sum),
#"Removed Columns1" = Table.RemoveColumns(#"Pivoted Column",{"BBN", "Total Sales", "Total Profit", "Total Bookings"}),
#"Replaced Value" = Table.ReplaceValue(#"Removed Columns1","Non-firm / Opportunity","Inactive/Delete",Replacer.ReplaceText,{"Booking Disposition (Rplan)"}),
#"Renamed Columns" = Table.RenameColumns(#"Replaced Value",{{"Distribution Channel_1", "Distribution Channel"}})
in
#"Renamed Columns"
)
in
result
This way of proceeding ("skipping" some calculation rather than stopping the whole evaluation) is hinted by Microsoft on their page here:
if if-condition then true-expression else false-expression
The true-expression is only evaluated if the if-condition evaluates to the value true.
The false-expression is only evaluated if the if-condition evaluates to the value false.

If else statement in Power Query throws error "Token Else expected"

I am pretty new to Power Query
I have two separate Power Querys that work in separate/stand-alone queries, but I want to use an if statement that that takes the value of a cell with a checkbox to use either the JSON or the CSV
I have a checkbox in a cell called useWhat then two other cells one called SourceLink which has a link to a JSON file and another called fp which is a local link to a folder with a CSV in it
I have the following statement, but I am getting an error right after the
SourceLink = Excel.CurrentWorkbook(){[Name="SourceLink"]}[Content]{0}[Column1],
the comma is highlighted in red.
Any assistance with this If statement is appreciated
Thanks
essentially I am trying to do this
let
#what =
if useWhat = "TRUE" then
import via JSON
else
import via CSV
end if
more code
what I have
let
#"what" =
if useWhat = "TRUE" then
SourceLink = Excel.CurrentWorkbook(){[Name="sourceLink"]}[Content]{0}[Column1],
Source = Json.Document(Web.Contents(sourceLink)),
Expanded=Table.FromRecords (Source[elements] ),
#"Removed Errors" = Table.RemoveRowsWithErrors(Expanded),
#"Added Index" = Table.AddIndexColumn(#"Removed Errors", "Index", 0, 1, Int64.Type),
#"Unpivoted Other Columns" = Table.UnpivotOtherColumns(#"Added Index", {"Index"}, "Attribute", "Value"),
#"Combine items in list" = Table.TransformColumns(#"Unpivoted Other Columns",{{"Value", each try if Value.Is(_, type list ) then Text.Combine(_,"|") else _ otherwise _, type text}}),
#"Pivoted Column" = Table.Pivot(#"Combine items in list", List.Distinct(#"Combine items in list"[Attribute]), "Attribute", "Value"),
#"Removed Columns" = Table.RemoveColumns(#"Pivoted Column",{"Index"}),
#"Changed Type" = Table.TransformColumnTypes(#"Removed Columns",List.Transform(Table.ColumnNames(#"Removed Columns"),each {_,type text}))
else
FilePath = Excel.CurrentWorkbook(){[Name="fp"]}[Content]{0}[TheFilePath],
Source = Csv.Document(File.Contents(FilePath),[Delimiter=",", Columns=18, Encoding=1252, QuoteStyle=QuoteStyle.None]),
#"Changed Type" = Table.TransformColumnTypes(Source,{{"Column1", type text}, {"Column2", type text}, {"Column3", type text}, {"Column4", type text}, {"Column5", type text}, {"Column6", type text}, {"Column7", type text}, {"Column8", type text}, {"Column9", type text}, {"Column10", type text}, {"Column11", type text}, {"Column12", type text}, {"Column13", type text}, {"Column14", type text}, {"Column15", type text}, {"Column16", type text}, {"Column17", type text}, {"Column18", type text}}),
#"Promoted Headers" = Table.PromoteHeaders(#"Changed Type", [PromoteAllScalars=true]),
#"Changed Type" = Table.TransformColumnTypes(#"Promoted Headers",List.Transform(Table.ColumnNames(#"Promoted Headers"),each {_,type text}))
end if
#"Appended Query" = Table.Combine({#"Changed Type", addMissingEmails}),
firstColumns = {"Id", "Label", "Email", "Project Name", "Type", "Segment", "Description", "First Name", "Last Name", "Email 2", "Email 3", "Initial Date", "Last Date", "Image"},
#"Reordered Columns" = Table.ReorderColumns(#"Appended Query",firstColumns & List.RemoveMatchingItems(Table.ColumnNames(#"Appended Query"),firstColumns),MissingField.Ignore),
#"Demoted Headers" = Table.DemoteHeaders(#"Reordered Columns"),
BulkReplaceStep = fBulkReplace(#"Demoted Headers", MyFindReplace, Table.ColumnNames(#"Demoted Headers")),
#"Replaced Value" = Table.ReplaceValue(BulkReplaceStep,"#(00A0)","",Replacer.ReplaceText,Table.ColumnNames(BulkReplaceStep)),
#"Replaced Value1" = Table.ReplaceValue(#"Replaced Value","#(0087)","",Replacer.ReplaceText,Table.ColumnNames(#"Replaced Value")),
#"Promoted Headers" = Table.PromoteHeaders(#"Replaced Value1", [PromoteAllScalars=true]),
#"Sorted Rows" = Table.Sort(#"Promoted Headers",{{"Label", Order.Ascending}}),
#"Trimmed Text" = Table.TransformColumns(#"Sorted Rows",{{"Id", Text.Trim, type text}}),
newMergedHeader = Table.ColumnNames(mergeHeaders){0},
#"Merged Columns" =
if newMergedHeader <> null then
Table.RenameColumns(
Table.CombineColumns(#"Trimmed Text",Table.Column(mergeHeaders,newMergedHeader),each Combiner.CombineTextByDelimiter("|", QuoteStyle.None)(List.Difference(List.RemoveNulls(_), {""})),"Merged"),{{"Merged",newMergedHeader}})
else #"Trimmed Text"
in
#"Merged Columns"
I'm coding completely blind but you need something like the following:
let
#"what" = if useWhat = "TRUE" then a else b,
a = let
SourceLink = Excel.CurrentWorkbook(){[Name="sourceLink"]}[Content]{0}[Column1],
Source = Json.Document(Web.Contents(sourceLink)),
Expanded=Table.FromRecords (Source[elements] ),
#"Removed Errors" = Table.RemoveRowsWithErrors(Expanded),
#"Added Index" = Table.AddIndexColumn(#"Removed Errors", "Index", 0, 1, Int64.Type),
#"Unpivoted Other Columns" = Table.UnpivotOtherColumns(#"Added Index", {"Index"}, "Attribute", "Value"),
#"Combine items in list" = Table.TransformColumns(#"Unpivoted Other Columns",{{"Value", each try if Value.Is(_, type list ) then Text.Combine(_,"|") else _ otherwise _, type text}}),
#"Pivoted Column" = Table.Pivot(#"Combine items in list", List.Distinct(#"Combine items in list"[Attribute]), "Attribute", "Value"),
#"Removed Columns" = Table.RemoveColumns(#"Pivoted Column",{"Index"}),
#"Changed Type" = Table.TransformColumnTypes(#"Removed Columns",List.Transform(Table.ColumnNames(#"Removed Columns"),each {_,type text}))
in #"Changed Type",
b = let
FilePath = Excel.CurrentWorkbook(){[Name="fp"]}[Content]{0}[TheFilePath],
Source = Csv.Document(File.Contents(FilePath),[Delimiter=",", Columns=18, Encoding=1252, QuoteStyle=QuoteStyle.None]),
#"Changed Type" = Table.TransformColumnTypes(Source,{{"Column1", type text}, {"Column2", type text}, {"Column3", type text}, {"Column4", type text}, {"Column5", type text}, {"Column6", type text}, {"Column7", type text}, {"Column8", type text}, {"Column9", type text}, {"Column10", type text}, {"Column11", type text}, {"Column12", type text}, {"Column13", type text}, {"Column14", type text}, {"Column15", type text}, {"Column16", type text}, {"Column17", type text}, {"Column18", type text}}),
#"Promoted Headers" = Table.PromoteHeaders(#"Changed Type", [PromoteAllScalars=true]),
#"Changed Type2" = Table.TransformColumnTypes(#"Promoted Headers",List.Transform(Table.ColumnNames(#"Promoted Headers"),each {_,type text}))
in #"Changed Type2",
#"Appended Query" = Table.Combine({#"what", addMissingEmails}),
firstColumns = {"Id", "Label", "Email", "Project Name", "Type", "Segment", "Description", "First Name", "Last Name", "Email 2", "Email 3", "Initial Date", "Last Date", "Image"},
#"Reordered Columns" = Table.ReorderColumns(#"Appended Query",firstColumns & List.RemoveMatchingItems(Table.ColumnNames(#"Appended Query"),firstColumns),MissingField.Ignore),
#"Demoted Headers" = Table.DemoteHeaders(#"Reordered Columns"),
BulkReplaceStep = fBulkReplace(#"Demoted Headers", MyFindReplace, Table.ColumnNames(#"Demoted Headers")),
#"Replaced Value" = Table.ReplaceValue(BulkReplaceStep,"#(00A0)","",Replacer.ReplaceText,Table.ColumnNames(BulkReplaceStep)),
#"Replaced Value1" = Table.ReplaceValue(#"Replaced Value","#(0087)","",Replacer.ReplaceText,Table.ColumnNames(#"Replaced Value")),
#"Promoted Headers" = Table.PromoteHeaders(#"Replaced Value1", [PromoteAllScalars=true]),
#"Sorted Rows" = Table.Sort(#"Promoted Headers",{{"Label", Order.Ascending}}),
#"Trimmed Text" = Table.TransformColumns(#"Sorted Rows",{{"Id", Text.Trim, type text}}),
newMergedHeader = Table.ColumnNames(mergeHeaders){0},
#"Merged Columns" =
if newMergedHeader <> null then
Table.RenameColumns(
Table.CombineColumns(#"Trimmed Text",Table.Column(mergeHeaders,newMergedHeader),each Combiner.CombineTextByDelimiter("|", QuoteStyle.None)(List.Difference(List.RemoveNulls(_), {""})),"Merged"),{{"Merged",newMergedHeader}})
else #"Trimmed Text"
in
#"Merged Columns"

Handling errors when expanding table in power query

I wish to expand a table that contains data from an invoked column pulling data from the web.
The issue is that not every row finds a desired result on the web and returns an error. Although I don't mind the result failing for this row it causes an issue when trying to expand the table because the table relies on all rows having the same captured headers for the expansion.
Below is an image showing the errors and the result of the expansion.
M Code:
let
Source = Excel.CurrentWorkbook(){[Name="Table1"]}[Content],
#"Changed Type" = Table.TransformColumnTypes(Source,{{"No.", Int64.Type}, {"CAS Number", type text}, {"Chemical name", type text}}),
#"Invoked Custom Function" = Table.AddColumn(#"Changed Type", "Fx GetBriefProfileLink", each #"Fx GetBriefProfileLink"([CAS Number])),
#"Expanded Fx GetBriefProfileLink" = Table.ExpandTableColumn(#"Invoked Custom Function", "Fx GetBriefProfileLink", {"Name", "Cas Number", "exported-column-briefProfileLink"}, {"Name", "Cas Number.1", "exported-column-briefProfileLink"})
in
#"Expanded Fx GetBriefProfileLink"
Fx GetBriefProfileLink: Data Public
(CAsNumberorName as text) =>
let
Source = Excel.Workbook(Web.Contents("https://echa.europa.eu/search-for-chemicals?p_p_id=disssimplesearch_WAR_disssearchportlet&p_p_lifecycle=2&p_p_state=normal&p_p_mode=view&p_p_resource_id=exportResults&p_p_cacheability=cacheLevelPage&_disssimplesearch_WAR_disssearchportlet_sessionCriteriaId=dissSimpleSearchSessionParam101401654440118533&_disssimplesearch_WAR_disssearchportlet_formDate=1654440118558&_disssimplesearch_WAR_disssearchportlet_sskeywordKey="&CAsNumberorName&"&_disssimplesearch_WAR_disssearchportlet_orderByCol=relevance&_disssimplesearch_WAR_disssearchportlet_orderByType=asc&_disssimplesearch_WAR_disssearchportlet_exportType=xls"))[Data]{0},
#"Removed Top Rows" = Table.Skip(Source,2),
#"Promoted Headers" = Table.PromoteHeaders(#"Removed Top Rows", [PromoteAllScalars=true]),
#"Changed Type" = Table.TransformColumnTypes(#"Promoted Headers",{{"Name", type text}, {"EC / List Number", type text}, {"Cas Number", type text}, {"Substance Information Page", type text}, {"exported-column-briefProfileLink", type text}, {"exported-column-obligationsLink", type text}}),
#"Removed Other Columns" = Table.SelectColumns(#"Changed Type",{"Name", "EC / List Number", "Cas Number", "exported-column-briefProfileLink"}),
#"Kept First Rows" = Table.FirstN(#"Removed Other Columns",1),
#"Removed Other Columns1" = Table.SelectColumns(#"Kept First Rows",{"Name", "Cas Number", "exported-column-briefProfileLink"})
in
#"Removed Other Columns1"
Sample Data:
No. CAS Number Chemical name
43 3380-30-1 5-chloro-2-(4-chlorphenoxy)phenol
44 03228-02-2 4-isopropyl-m-cresol
45 89-83-8 Thymol
46 60207-90-1 Propiconazole
47 5395-50-6 Tetrahydro-1,3,4,6-tetrakis(hydroxymethyl)imidazo[4,5-d]imidazole-2,5(1H,3H)-dione
48 15630-89-4 Sodium percarbonate
49 027176-87-0 Dodecylbenzenesulfonic acid
50 001344-09-8 Sodium silicate
It appears to be a glitch in PQ looking online however I am wondering if there are any workarounds.
My desired output is simply the same expansion but rows with errors just appear empty in the expanded section.
You just replace errors with null before your expansion.
let
Source = Excel.CurrentWorkbook(){[Name="Table1"]}[Content],
#"Changed Type" = Table.TransformColumnTypes(Source,{{"No.", Int64.Type}, {"CAS Number", type text}, {"Chemical name", type text}, {"Column1", type text}}),
#"Merged Columns" = Table.CombineColumns(#"Changed Type",{"Chemical name", "Column1"},Combiner.CombineTextByDelimiter(" ", QuoteStyle.None),"Merged"),
#"Renamed Columns" = Table.RenameColumns(#"Merged Columns",{{"Merged", "Chemical name"}}),
#"Added Custom" = Table.AddColumn(#"Renamed Columns", "Custom", each #"Fx GetBriefProfileLink"([CAS Number])),
#"Replaced Errors" = Table.ReplaceErrorValues(#"Added Custom", {{"Custom", null}}),
#"Expanded Custom" = Table.ExpandTableColumn(#"Replaced Errors", "Custom", {"Name", "Cas Number", "exported-column-briefProfileLink"}, {"Custom.Name", "Custom.Cas Number", "Custom.exported-column-briefProfileLink"})
in
#"Expanded Custom"

Analyzing multiple columns in power query

I wish to generate the Most common, Highest and Lowest values from the following data:
I have also added an additional column to handle text comments.
M Code so far:
let
Source = Excel.CurrentWorkbook(){[Name="Table1"]}[Content],
#"Changed Type" = Table.TransformColumnTypes(Source,{{"Column1", type any}, {"Column2", type any}, {"Column3", type any}, {"Column4", type any}, {"Column5", type any}, {"Column6", type any}}),
#"Added Index" = Table.AddIndexColumn(#"Changed Type", "Index", 0, 1, Int64.Type),
#"Added Custom" = Table.AddColumn(#"Added Index", "Most Common", each List.Mode(
Record.ToList(
Table.SelectColumns(#"Added Index",
List.RemoveFirstN(
Table.ColumnNames(#"Changed Type"))){[Index]}))),
#"Added Custom1" = Table.AddColumn(#"Added Custom", "Highest", each List.Max(
Record.ToList(
Table.SelectColumns(#"Added Index",
List.RemoveFirstN(
Table.ColumnNames(#"Changed Type"))){[Index]}))),
#"Added Custom2" = Table.AddColumn(#"Added Custom1", "Min", each List.Min(
Record.ToList(
Table.SelectColumns(#"Added Index",
List.RemoveFirstN(
Table.ColumnNames(#"Changed Type"))){[Index]}))),
#"Removed Other Columns" = Table.SelectColumns(#"Added Custom2",{"Most Common", "Highest", "Min"})
in
#"Removed Other Columns"
As shown in the image this isn't quite right for the following errors:
When there is an equal split the most common returns null (expected)
Text pulls through as the highest value (not expected)
Will be working on this but any suggestions are appreciated.
Assuming:
You made a mistake on row 7 and 'Not Limited' should be amongst the most frequent values;
You also want to know that 33 is the lowest value in row 12, not just the highest;
You can possibly have multiple unique text values you'd like to concatenate.
let
Source = Excel.CurrentWorkbook(){[Name="Tabel1"]}[Content],
#"Changed Type" = Table.TransformColumnTypes(Source,{{"Column1", type text}, {"Column2", type text}, {"Column3", type text}, {"Column4", type text}, {"Column5", type text}, {"Column6", type text}}),
#"Added Custom1" = Table.AddColumn(#"Changed Type", "Lists", each Text.Split(Text.Combine({[Column1],[Column2],[Column3],[Column4],[Column5],[Column6]},"|"),"|")),
#"Added Custom" = Table.AddColumn(#"Added Custom1", "Most Common", each Text.Combine(List.Modes([Lists]),",")),
#"Added Custom2" = Table.AddColumn(#"Added Custom", "Highest", each List.Max(List.Transform([Lists], each try Number.FromText(_) otherwise null))),
#"Added Custom3" = Table.AddColumn(#"Added Custom2", "Lowest", each List.Min(List.Transform([Lists], each try Number.FromText(_) otherwise null))),
#"Added Custom4" = Table.AddColumn(#"Added Custom3", "Text Comments", each Text.Combine(List.Distinct(List.RemoveMatchingItems(List.Transform([Lists], each try if Number.FromText(_) <>"" then "" else "" otherwise (_)),{""})),",")),
#"Replaced Errors" = Table.ReplaceErrorValues(#"Added Custom4", {{"Highest", null}, {"Lowest", null}}),
#"Removed Columns" = Table.RemoveColumns(#"Replaced Errors",{"Column1", "Column2", "Column3", "Column4", "Column5", "Column6", "Lists"})
in
#"Removed Columns"
Steps:
All columns to "Type Text";
Created an helper column with lists of values after Text.Combine & Text.Split combo;
Used List.Modes to return most common values inside a Text.Combine to return 'Most Common';
Used List.Max and List.Min in combination with List.Transform to return 'Highest' and 'Lowest' value;
Used a combination of Text.Combine, List.Distinct, List.RemoveMatchingItems and List.Transform to return only unique actual text values;
Removed columns1-6 and helper and replaced errors with 'null'.

How can I check if a Power Query is complete using batch or Powershell?

I have a batch script which downloads some information to a csv file, then opens an Excel spreadsheet, which uses this csv as the source for a Power Query which refreshes whenever the spreadsheet is opened. After it has loaded information from the csv, I want the batch script to delete the csv.
To check if a file is locked for writing, I can use this:
:deletecsv
2>nul (
>>info.csv(call )
) && del info.csv || (goto :deletecsv)
However, surprisingly Excel does not seem to lock files for writing while a power query is searching them. At any rate my script just deletes the csv straight away, leaving an empty file for Excel to look at.
Can I detect what applications are reading the csv? Another route would be for the Power Query to delete the original file at the end, but I'm not aware that's possible. I could just timeout for an arbitrary duration, but in future there may be a lot more to download so not sure how long query will take. VBA is not an option.
The Power Query:
let
Source = Csv.Document(File.Contents("C:\Users\user\api\info.csv"),[Delimiter=",", Columns=31, Encoding=1252, QuoteStyle=QuoteStyle.Csv]),
#"Promoted Headers" = Table.PromoteHeaders(Source, [PromoteAllScalars=true]),
#"Changed Type" = Table.TransformColumnTypes(#"Promoted Headers",{{"activityid", Int64.Type}, {"activityenddate", type datetime}, {"activityendtime", type time}, {"activityfrequency", Int64.Type}, {"activitystartdate", type datetime}, {"activitystarttime", type time}, {"activitysubtype", type text}, {"activitytype", type text}, {"ambassadorenddate", type datetime}, {"ambassadorstartdate", type datetime}, {"attendedstudentscount", Int64.Type}, {"contacthours", type number}, {"customdata", type text}, {"deliveredbytype", Int64.Type}, {"displaysummary", type text}, {"eventtitle", type text}, {"feyear", Int64.Type}, {"heatlevelofintervention", Int64.Type}, {"heidescriptor", type text}, {"interaction", Int64.Type}, {"locationid", Int64.Type}, {"notes", type text}, {"projectidentifier", type text}, {"recordcreateddate", type datetime}, {"recordupdateddate", type datetime}, {"systemnotes", type text}, {"totalparents", Int64.Type}, {"totalstaff", Int64.Type}, {"yeargroups", type text}, {"activitybeneficiaryinstitutions", type text}}),
#"Split Column by Delimiter" = Table.SplitColumn(#"Changed Type", "yeargroups", Splitter.SplitTextByDelimiter(";", QuoteStyle.Csv), {"yeargroups.1", "yeargroups.2", "yeargroups.3", "yeargroups.4"}),
#"Changed Type1" = Table.TransformColumnTypes(#"Split Column by Delimiter",{{"yeargroups.1", type text}, {"yeargroups.2", type text}, {"yeargroups.3", type text}, {"yeargroups.4", type text}, {"activityenddate", type date}, {"activitystartdate", type date}}),
#"Split Column by Delimiter1" = Table.SplitColumn(#"Changed Type1", "activitysubtype", Splitter.SplitTextByDelimiter(";", QuoteStyle.Csv), {"activitysubtype.1", "activitysubtype.2", "activitysubtype.3"}),
#"Changed Type2" = Table.TransformColumnTypes(#"Split Column by Delimiter1",{{"activitysubtype.1", type text}, {"activitysubtype.2", type text}, {"activitysubtype.3", type text}}),
#"Removed Columns" = Table.RemoveColumns(#"Changed Type2",{"activitysubtype.1"}),
#"Replaced Value" = Table.ReplaceValue(#"Removed Columns","name=","",Replacer.ReplaceText,{"activitysubtype.2"}),
#"Removed Columns1" = Table.RemoveColumns(#"Replaced Value",{"activitysubtype.3"}),
#"Split Column by Delimiter2" = Table.SplitColumn(#"Removed Columns1", "activitytype", Splitter.SplitTextByDelimiter(";", QuoteStyle.Csv), {"activitytype.1", "activitytype.2", "activitytype.3"}),
#"Extracted Text Between Delimiters" = Table.TransformColumns(#"Split Column by Delimiter2", {{"customdata", each Text.BetweenDelimiters(_, "}", "Learner Outcome=", {0, RelativePosition.FromEnd}, {0, RelativePosition.FromEnd}), type text}}),
#"Extracted Text Before Delimiter" = Table.TransformColumns(#"Extracted Text Between Delimiters", {{"customdata", each Text.BeforeDelimiter(_, "event?="), type text}}),
#"Replaced Value1" = Table.ReplaceValue(#"Extracted Text Before Delimiter","#{Is this event a large scale carousel ","",Replacer.ReplaceText,{"customdata"}),
#"Split Column by Delimiter3" = Table.SplitColumn(#"Replaced Value1", "customdata", Splitter.SplitTextByDelimiter(";", QuoteStyle.Csv), {"customdata.1", "customdata.2", "customdata.3", "customdata.4", "customdata.5", "customdata.6", "customdata.7"}),
#"Changed Type3" = Table.TransformColumnTypes(#"Split Column by Delimiter3",{{"activitytype.1", type text}, {"activitytype.2", type text}, {"activitytype.3", type text}, {"customdata.1", type text}, {"customdata.2", type text}, {"customdata.3", type text}, {"customdata.4", type text}, {"customdata.5", type text}, {"customdata.6", type text}, {"customdata.7", type text}}),
#"Renamed Columns" = Table.RenameColumns(#"Changed Type3",{{"customdata.1", "Outcome"}, {"customdata.2", "Theme"}}),
#"Replaced Value2" = Table.ReplaceValue(#"Renamed Columns"," Progression Programme Theme=","",Replacer.ReplaceText,{"Theme"}),
#"Removed Columns2" = Table.RemoveColumns(#"Replaced Value2",{"customdata.3"}),
#"Replaced Value3" = Table.ReplaceValue(#"Removed Columns2","Platform=","",Replacer.ReplaceText,{"customdata.4"}),
#"Removed Columns3" = Table.RemoveColumns(#"Replaced Value3",{"customdata.5", "customdata.6"}),
#"Extracted Text After Delimiter" = Table.TransformColumns(#"Removed Columns3", {{"customdata.7", each Text.AfterDelimiter(_, "="), type text}}),
#"Renamed Columns1" = Table.RenameColumns(#"Extracted Text After Delimiter",{{"customdata.4", "Platform"}, {"customdata.7", "Phase"}}),
#"Extracted Text Between Delimiters1" = Table.TransformColumns(#"Renamed Columns1", {{"heidescriptor", each Text.BetweenDelimiters(_, "name=", ";"), type text}}),
#"Removed Columns4" = Table.RemoveColumns(#"Extracted Text Between Delimiters1",{"notes"}),
#"Changed Type4" = Table.TransformColumnTypes(#"Removed Columns4",{{"recordcreateddate", type date}, {"recordupdateddate", type date}}),
#"Removed Columns5" = Table.RemoveColumns(#"Changed Type4",{"systemnotes", "yeargroups.1", "yeargroups.2", "yeargroups.4"}),
#"Replaced Value4" = Table.ReplaceValue(#"Removed Columns5","name=","",Replacer.ReplaceText,{"yeargroups.3"}),
#"Extracted Text Between Delimiters2" = Table.TransformColumns(#"Replaced Value4", {{"activitybeneficiaryinstitutions", each Text.BetweenDelimiters(_, " name=", ";"), type text}}),
#"Removed Columns6" = Table.RemoveColumns(#"Extracted Text Between Delimiters2",{"activitytype.1", "activitytype.3"}),
#"Replaced Value5" = Table.ReplaceValue(#"Removed Columns6","name=","",Replacer.ReplaceText,{"activitytype.2"}),
#"Extracted Text Between Delimiters3" = Table.TransformColumns(#"Replaced Value5", {{"institutiongroup", each Text.BetweenDelimiters(_, " name=", ";"), type text}})
in
#"Extracted Text Between Delimiters3"

Resources