diff --git a/DESCRIPTION b/DESCRIPTION index 0ef8a2b..60fe71e 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -3,21 +3,23 @@ Title: Interface to the municode.com API Version: 0.0.0.9000 Authors@R: person("Will", "Curran-Groome", , "wcurrangroome@gmail.com", role = c("aut", "cre")) -Description: Access and programmatically navigated ordinances and associated metadata relating to regulatory documents hosted on municode.com. +Description: Access and programmatically navigate ordinances and associated metadata relating to regulatory documents hosted on municode.com. License: MIT + file LICENSE Encoding: UTF-8 Roxygen: list(markdown = TRUE) -RoxygenNote: 7.3.3 Imports: dplyr, httr2, janitor, magrittr, + purrr, rlang, stringr, tibble, - tidyr + tidyr, + utils Suggests: testthat (>= 3.0.0) Config/testthat/edition: 3 URL: https://wcurrangroome.github.io/municoder/ +Config/roxygen2/version: 8.0.0 diff --git a/NAMESPACE b/NAMESPACE index 39b28da..8d1a4df 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,9 +1,11 @@ # Generated by roxygen2: do not edit by hand +export(get_changed_sections) export(get_client_metadata) export(get_client_products) export(get_clients_in_state) export(get_codes_toc) +export(get_codes_tree) export(get_current_version) export(get_jurisdiction_products) export(get_ordinance_ancestors) diff --git a/R/clients.R b/R/clients.R index d4abc53..2b3d6b1 100644 --- a/R/clients.R +++ b/R/clients.R @@ -13,18 +13,20 @@ #' get_client_metadata("VA", "Alexandria") #' } get_client_metadata <- function(state_abbreviation, client_name) { - result <- - build_endpoint( - domain = "Clients", - subdomain = "name", - parameters = c(stateAbbr = state_abbreviation, clientName = client_name)) %>% - get_endpoint() %>% - tibble::enframe() %>% - tidyr::pivot_wider() %>% - transform_nested_state() %>% - janitor::clean_names() + ## The Clients/name endpoint now returns HTTP 404 for every request, so this + ## filters the (still working) per-state client listing by name instead. + state_clients <- get_clients_in_state(state_abbreviation) + + result <- state_clients %>% + dplyr::filter(stringr::str_to_lower(client_name) == stringr::str_to_lower(!!client_name)) + + if (nrow(result) == 0) { + stop( + "No Municode client named '", client_name, "' in state '", + state_abbreviation, "'.") + } - return(result) + result } #' Return metadata for all Municode clients in a given state @@ -51,7 +53,7 @@ get_clients_in_state <- function(state_abbreviation) { janitor::clean_names() %>% clean_client_columns() - return(result) + result } #' Obtain products available for a jurisdiction @@ -74,7 +76,6 @@ get_client_products <- function(client_id) { tibble::enframe() %>% tidyr::pivot_wider() - # Check if codes column exists if (!"codes" %in% names(raw_result)) { stop(sprintf( "Failed to fetch products for client_id %s. Client may not exist or have no products available.", @@ -89,5 +90,5 @@ get_client_products <- function(client_id) { tidyr::unnest_wider(codes) %>% janitor::clean_names() - return(result) + result } diff --git a/R/codes.R b/R/codes.R index d5b81f3..90df425 100644 --- a/R/codes.R +++ b/R/codes.R @@ -1,9 +1,59 @@ #' @title Code-related API functions (current ordinance versions) +#' Fetch and reshape the raw CodesContent response for a node +#' @param node_id A unique identifier for a node within the specified product +#' @param product_id A unique identifier for a product +#' @param job_id Optional job identifier; required when `show_changes = TRUE` +#' @param past_job_id Optional prior job to diff against; required when +#' `show_changes = TRUE` +#' @param show_changes If `TRUE`, request redline markup comparing `job_id` +#' against `past_job_id` +#' @return A one-row tibble with one column per top-level response field +#' @keywords internal +fetch_section_result <- function(node_id, product_id, job_id = NULL, + past_job_id = NULL, show_changes = FALSE) { + parameters <- c(nodeId = node_id, productId = product_id) + + if (isTRUE(show_changes)) { + if (is.null(job_id) || is.null(past_job_id)) { + stop("`show_changes = TRUE` requires both `job_id` and `past_job_id`.", + call. = FALSE) + } + parameters <- c(parameters, + jobId = job_id, pastJobId = past_job_id, showChanges = "true") + } else if (!is.null(job_id)) { + parameters <- c(parameters, jobId = job_id) + } + + build_endpoint(domain = "CodesContent", parameters = parameters) %>% + get_endpoint() %>% + tibble::enframe() %>% + tidyr::pivot_wider() +} + +#' Extract the Docs payload (id, title, content) from a CodesContent response +#' @param result A reshaped CodesContent response from `fetch_section_result()` +#' @return A tibble with cleaned `id`, `title`, and `content` columns +#' @keywords internal +extract_section_docs <- function(result) { + result %>% + dplyr::select(Docs) %>% + tidyr::unnest_longer(Docs) %>% + tidyr::unnest_wider(Docs) %>% + janitor::clean_names() +} + #' Get text and metadata for a given node within a given ordinance #' @description Corresponds to, for example: https://api.municode.com/CodesContent?jobId=426172&nodeId=THZOORALVI&productId=12429 #' @param node_id A unique identifier for a node within the specified product (ordinance) #' @param product_id A unique identifier for a product +#' @param job_id Optional job identifier; required when `show_changes = TRUE` +#' @param past_job_id Optional prior job identifier to diff against; required +#' when `show_changes = TRUE` +#' @param show_changes If `TRUE`, request a redline comparison of `job_id` +#' against `past_job_id`. Note that `get_section_text()` strips HTML, which +#' discards the insertion/deletion markup; use [get_section_html()] to retain +#' the redline in the `content_html` column. #' #' @returns A dataframe with the content and metadata pertaining to a specific node within an ordinance #' @export @@ -12,59 +62,55 @@ #' \dontrun{ #' get_section_text(node_id = "SUHITA", product_id = "12429") #' } -get_section_text <- function(node_id = NULL, product_id) { - result <- - build_endpoint( - domain = "CodesContent", - parameters = c(nodeId = node_id, productId = product_id)) %>% - get_endpoint() %>% - tibble::enframe() %>% - tidyr::pivot_wider() +get_section_text <- function(node_id = NULL, product_id, job_id = NULL, + past_job_id = NULL, show_changes = FALSE) { + result <- fetch_section_result(node_id, product_id, job_id = job_id, + past_job_id = past_job_id, show_changes = show_changes) - docs <- result %>% - dplyr::select(Docs) %>% - tidyr::unnest_longer(Docs) %>% - tidyr::unnest_wider(Docs) %>% - janitor::clean_names() %>% + docs <- extract_section_docs(result) %>% dplyr::transmute( id, node_type = "current", heading = title, content = clean_html_content(content)) + ## Guard on nrow(): a node with no next/previous sibling unnests to a + ## zero-row tibble, and length() would count columns (always >= 1) and + ## wrongly enter the branch. Empty siblings collapse to NULL so they + ## contribute no spurious all-NA columns to the bound result. next_node <- result %>% dplyr::select(NextNode) %>% tidyr::unnest_longer(NextNode) - if (length(next_node) > 0) { - next_node <- next_node %>% + next_node <- if (nrow(next_node) > 0) { + next_node %>% tidyr::pivot_wider(names_from = NextNode_id, values_from = NextNode) %>% tidyr::unnest_wider(Data) %>% dplyr::mutate(dplyr::across(dplyr::everything(), unlist)) %>% - janitor::clean_names() + janitor::clean_names() %>% + dplyr::mutate(node_type = "next") + } else { + NULL } previous_node <- result %>% dplyr::select(PrevNode) %>% tidyr::unnest_longer(PrevNode) - if (length(previous_node) > 0) { - previous_node <- previous_node %>% + previous_node <- if (nrow(previous_node) > 0) { + previous_node %>% tidyr::pivot_wider(names_from = PrevNode_id, values_from = PrevNode) %>% tidyr::unnest_wider(Data) %>% dplyr::mutate(dplyr::across(dplyr::everything(), unlist)) %>% - janitor::clean_names() + janitor::clean_names() %>% + dplyr::mutate(node_type = "previous") + } else { + NULL } - result <- - dplyr::bind_rows( - docs, - next_node %>% dplyr::mutate(node_type = "next"), - previous_node %>% dplyr::mutate(node_type = "previous")) %>% + dplyr::bind_rows(docs, next_node, previous_node) %>% dplyr::select(id, heading, node_type, content, dplyr::everything()) - - return(result) - } +} #' Get the Table of Contents for an ordinance #' @description Corresponds to, for example: https://api.municode.com/codesToc?jobId=426172&productId=12429 @@ -95,8 +141,65 @@ get_codes_toc <- function(job_id, product_id) { dplyr::mutate(dplyr::across(.cols = c(dplyr::where(is.list), -Data), unlist)) %>% janitor::clean_names() %>% dplyr::select(-c(data, node_depth, doc_order_id)) +} + +#' Recursively flatten a codesToc tree node into a tidy dataframe +#' @param node A node list from the `codesToc/fullTree` response +#' @return A dataframe with one row per node in the subtree rooted at `node` +#' @keywords internal +flatten_codes_tree <- function(node) { + or_na <- function(x) if (is.null(x)) NA else x + data <- node$Data - return(result) + this_row <- tibble::tibble( + id = as.character(or_na(node$Id)), + heading = as.character(or_na(node$Heading)), + node_depth = or_na(node$NodeDepth), + parent_id = as.character(or_na(node$ParentId)), + doc_order_id = or_na(node$DocOrderId), + has_children = or_na(node$HasChildren), + node_key = as.character(or_na(data$NodeKey)), + is_updated = or_na(data$IsUpdated), + is_amended = or_na(data$IsAmended), + has_amended_descendant = or_na(data$HasAmendedDescendant), + compare_status = or_na(data$CompareStatus), + doc_type = or_na(data$DocType)) + + if (length(node$Children) == 0) { + return(this_row) + } + + dplyr::bind_rows(c(list(this_row), purrr::map(node$Children, flatten_codes_tree))) +} + +#' Get the full table of contents tree for an ordinance in a single request +#' @description Corresponds to, for example: https://api.municode.com/codesToc/fullTree?jobId=494092&nodeId=15441&productId=15441 +#' Unlike [get_section_children()], which returns one level of the hierarchy +#' per call, this returns the entire nested tree beneath `node_id` in a single +#' request and flattens it to a tidy dataframe (one row per node). Per-node +#' amendment flags are included. +#' @param job_id A unique identifier for a job +#' @param node_id A node to use as the root of the returned tree. Pass the +#' `product_id` to retrieve the whole code. +#' @param product_id A unique identifier for a product +#' +#' @returns A dataframe with one row per node and the columns `id`, `heading`, +#' `node_depth`, `parent_id`, `doc_order_id`, `has_children`, `node_key`, +#' `is_updated`, `is_amended`, `has_amended_descendant`, `compare_status`, +#' and `doc_type`. +#' @export +#' +#' @examples +#' \dontrun{ +#' get_codes_tree(job_id = 494092, node_id = 15441, product_id = 15441) +#' } +get_codes_tree <- function(job_id, node_id, product_id) { + build_endpoint( + domain = "codesToc", + subdomain = "fullTree", + parameters = c(jobId = job_id, nodeId = node_id, productId = product_id)) %>% + get_endpoint() %>% + flatten_codes_tree() } #' Get raw HTML content and markdown for a given node within a given ordinance @@ -106,6 +209,12 @@ get_codes_toc <- function(job_id, product_id) { #' sections that contain HTML tables (e.g., dimensional standards). #' @param node_id A unique identifier for a node within the specified product (ordinance) #' @param product_id A unique identifier for a product +#' @param job_id Optional job identifier; required when `show_changes = TRUE` +#' @param past_job_id Optional prior job identifier to diff against; required +#' when `show_changes = TRUE` +#' @param show_changes If `TRUE`, request a redline comparison of `job_id` +#' against `past_job_id`. The insertion/deletion markup is preserved in the +#' returned `content_html` column. #' #' @returns A dataframe with the raw HTML content and metadata for the specified node. #' Columns: `id`, `heading`, `node_type`, `content_html`, `content_markdown`. @@ -115,28 +224,17 @@ get_codes_toc <- function(job_id, product_id) { #' \dontrun{ #' get_section_html(node_id = "SUHITA", product_id = "12429") #' } -get_section_html <- function(node_id = NULL, product_id) { - result <- - build_endpoint( - domain = "CodesContent", - parameters = c(nodeId = node_id, productId = product_id)) %>% - get_endpoint() %>% - tibble::enframe() %>% - tidyr::pivot_wider() - - docs <- result %>% - dplyr::select(Docs) %>% - tidyr::unnest_longer(Docs) %>% - tidyr::unnest_wider(Docs) %>% - janitor::clean_names() %>% +get_section_html <- function(node_id = NULL, product_id, job_id = NULL, + past_job_id = NULL, show_changes = FALSE) { + fetch_section_result(node_id, product_id, job_id = job_id, + past_job_id = past_job_id, show_changes = show_changes) %>% + extract_section_docs() %>% dplyr::transmute( id, node_type = "current", heading = title, content_html = content, content_markdown = html_to_markdown(content)) - - return(docs) } #' Get information about a given node's ancestor(s) @@ -163,8 +261,6 @@ get_section_ancestors <- function(job_id, node_id, product_id) { node, id = node_id, ancestor_id = ancestors_id) - - return(result) } #' Get information about a given node's children @@ -197,8 +293,8 @@ get_section_children <- function(job_id, node_id, product_id) { tidyr::unnest_wider(value) if ("Data" %in% names(result)) { - result <- result %>% tidyr::unnest_wider(Data) + result <- tidyr::unnest_wider(result, Data) } - result %>% janitor::clean_names() + janitor::clean_names(result) } diff --git a/R/jobs.R b/R/jobs.R index a392397..e1c046d 100644 --- a/R/jobs.R +++ b/R/jobs.R @@ -22,12 +22,10 @@ get_version_history <- function(product_id) { ), call. = FALSE) } - result <- raw_result %>% + raw_result %>% tibble::enframe() %>% tidyr::unnest_wider(value) %>% janitor::clean_names() - - return(result) } #' Get the current version of a product @@ -36,35 +34,97 @@ get_version_history <- function(product_id) { #' @param product_id A unique identifier for a product #' @export get_current_version <- function(product_id) { - ## Some products (e.g., Salem, OR / product_id 15441) return an empty - ## `Product` object from /Jobs/latest, which means downstream nested - ## fields like `ProductContentType` are absent. Unnest each nested - ## column only when it exists and has content. - safe_unnest_wider <- function(df, col, names_sep = NULL) { - if (!col %in% names(df)) return(df) - val <- df[[col]] - if (is.list(val) && all(lengths(val) == 0)) { - df[[col]] <- NULL - return(df) - } - tidyr::unnest_wider(df, dplyr::all_of(col), names_sep = names_sep) - } - - result <- + raw_result <- build_endpoint( domain = "Jobs", subdomain = "latest") %>% stringr::str_c("/", product_id) %>% - get_endpoint() %>% - tibble::enframe() %>% - tidyr::pivot_wider() %>% - safe_unnest_wider("Product", names_sep = "") %>% - safe_unnest_wider("ProductContentType", names_sep = "_") %>% - safe_unnest_wider("ProductFeatures") %>% - safe_unnest_wider("ProductClient") %>% - safe_unnest_wider("State") %>% + get_endpoint() + + if (length(raw_result) == 0 || is.null(raw_result)) { + stop(sprintf( + "Failed to fetch current version for product_id %s. Product may not exist.", + product_id + ), call. = FALSE) + } + + ## Drop NULL elements so enframe()/pivot_wider() yield one row per field; the + ## Municode response schema is not stable (nested objects such as Product, + ## ProductContentType, ProductFeatures, ProductClient, and State are sometimes + ## absent or NULL), so unnest only the nested columns that are actually present. + result <- raw_result %>% + purrr::compact() %>% + tibble::enframe() %>% + tidyr::pivot_wider() + + nested_cols <- c( + Product = "", + ProductContentType = "_", + ProductFeatures = NA_character_, + ProductClient = NA_character_, + State = NA_character_) + + for (col in names(nested_cols)) { + if (col %in% names(result) && is.list(result[[col]])) { + sep <- nested_cols[[col]] + if (is.na(sep)) { + result <- tidyr::unnest_wider(result, dplyr::all_of(col)) + } else { + result <- tidyr::unnest_wider(result, dplyr::all_of(col), names_sep = sep) + } + } + } + + result %>% janitor::clean_names() %>% dplyr::mutate(dplyr::across(.cols = dplyr::where(is.list), unlist)) +} - return(result) +#' List the sections that changed in a given version of an ordinance +#' @description Corresponds to, for example: https://api.municode.com/CodesChangedDocs?jobId=494092&productId=15441 +#' Returns the set of sections (nodes) that were added or amended in the +#' specified job relative to the preceding version. Useful for tracking +#' ordinance changes over time; pair with [get_version_history()] to walk +#' versions and [get_section_html()] (with `show_changes = TRUE`) to retrieve +#' the redline for each changed node. +#' @param job_id A unique identifier for a job +#' @param product_id A unique identifier for a product +#' +#' @returns A dataframe with one row per changed section and the columns +#' `node_id`, `title`, `compare_status` (an undocumented change-type code), +#' and `ancestor_path` (the node's breadcrumb, titles joined by `" > "`). +#' Returns a zero-row dataframe when the version introduced no changes. +#' @export +#' +#' @examples +#' \dontrun{ +#' get_changed_sections(job_id = 494092, product_id = 15441) +#' } +get_changed_sections <- function(job_id, product_id) { + raw_result <- + build_endpoint( + domain = "CodesChangedDocs", + parameters = c(jobId = job_id, productId = product_id)) %>% + get_endpoint() + + if (length(raw_result) == 0) { + return(tibble::tibble( + node_id = character(), + title = character(), + compare_status = integer(), + ancestor_path = character())) + } + + raw_result %>% + tibble::enframe(name = NULL) %>% + tidyr::unnest_wider(value) %>% + janitor::clean_names() %>% + dplyr::mutate( + ancestor_path = purrr::map_chr(ancestors, function(a) + paste(purrr::map_chr(a, function(x) as.character(x$Title)), collapse = " > "))) %>% + dplyr::transmute( + node_id = as.character(node_id), + title = as.character(title), + compare_status, + ancestor_path) } diff --git a/R/municoder-package.R b/R/municoder-package.R index a1bb596..a8bceb6 100644 --- a/R/municoder-package.R +++ b/R/municoder-package.R @@ -12,7 +12,8 @@ utils::globalVariables(c( ".", "Ancestors", "Children", "Client", "Client_State", "ContentType", "Data", "Docs", "Heading", "Id", "NextNode", "NextNode_id", "Node", "PrevNode", "PrevNode_id", "Product", "ProductClient", "ProductContentType", "ProductFeatures", "State", - "ancestor_id", "ancestors", "ancestors_id", "children", "client_classification_id", + "ancestor_id", "ancestor_path", "ancestors", "ancestors_id", "children", + "client_classification_id", "compare_status", "client_id", "client_name", "client_pop_range_id", "client_show_advance_sheet", "codes", "content", "data", "doc_order_id", "docs_content", "heading", "id", "name", "node", "node_depth", "node_id", "node_type", "product_id", "product_name", diff --git a/R/ordinances.R b/R/ordinances.R index 6f05c60..b520e08 100644 --- a/R/ordinances.R +++ b/R/ordinances.R @@ -2,40 +2,39 @@ #' Return information about all ordinances of a given type for a given year #' @description Corresponds to, for example: https://api.municode.com/CoreContent/Ordinances?nodeId=2023&productId=12429 +#' @param node_id The year for which ordinances are requested. Optional; when +#' omitted the response is the top-level index rather than a list of ordinances. #' @param product_id A unique code identifying a product (e.g., a zoning ordinance) -#' @param node_id The year for which ordinances are requested #' -#' @returns A dataframe comprising ordinances +#' @returns A dataframe comprising ordinances. The shape depends on `node_id`: +#' when the response contains nested list elements (typically when `node_id` +#' is supplied) each ordinance becomes a row; otherwise the top-level fields +#' are pivoted into a single row. #' @export -list_ordinances <- function(product_id, node_id = NULL) { - endpoint <- build_endpoint( +list_ordinances <- function(node_id = NULL, product_id) { + result <- build_endpoint( domain = "CoreContent", subdomain = "Ordinances", - parameters = c(productId = product_id, nodeId = node_id)) - - result <- endpoint %>% + parameters = c(productId = product_id, nodeId = node_id)) %>% get_endpoint() %>% tibble::enframe() - # Handle different response structures based on whether node_id is provided - if (any(sapply(result$value, is.list))) { - result <- result %>% + if (any(purrr::map_lgl(result$value, is.list))) { + result %>% tidyr::unnest_longer(value) %>% tidyr::unnest_wider(value) %>% janitor::clean_names() } else { - result <- result %>% + result %>% tidyr::pivot_wider() %>% janitor::clean_names() } - - return(result) } #' Obtain metadata about a given ordinance over time #' @description Corresponds to, for example: https://api.municode.com/ordinancesToc?nodeId=2023&productId=12429 -#' @param product_id A unique identifier for a product #' @param node_id A unique identifier for a node within the specified product (ordinance) +#' @param product_id A unique identifier for a product #' #' @export #' @@ -43,33 +42,29 @@ list_ordinances <- function(product_id, node_id = NULL) { #' \dontrun{ #' get_ordinances_toc(node_id = 2023, product_id = 12429) #' } -get_ordinances_toc <- function(product_id, node_id = NA) { - result <- - build_endpoint( - domain = "ordinancesToc", - parameters = c(nodeId = node_id, productId = product_id)) %>% +get_ordinances_toc <- function(node_id = NULL, product_id) { + build_endpoint( + domain = "ordinancesToc", + parameters = c(nodeId = node_id, productId = product_id)) %>% get_endpoint() %>% tibble::enframe() %>% tidyr::pivot_wider() %>% janitor::clean_names() %>% tidyr::unnest_wider(data) %>% dplyr::mutate(dplyr::across(.cols = -children, unlist)) - - return(result) } #' Get the ancestors of a given node in an ordinance's Table of Contents #' @description Corresponds to, for example: https://api.municode.com/ordinancesToc/breadcrumb?nodeId=2023&productId=12429 -#' @param product_id A unique identifier for a product #' @param node_id A unique identifier for a node within the specified product (ordinance) +#' @param product_id A unique identifier for a product #' #' @export -get_ordinance_ancestors <- function(product_id, node_id) { - result <- - build_endpoint( - domain = "ordinancesToc", - subdomain = "breadcrumb", - parameters = c(nodeId = node_id, productId = product_id)) %>% +get_ordinance_ancestors <- function(node_id, product_id) { + build_endpoint( + domain = "ordinancesToc", + subdomain = "breadcrumb", + parameters = c(nodeId = node_id, productId = product_id)) %>% get_endpoint() %>% tibble::enframe() %>% tidyr::pivot_wider() %>% @@ -78,25 +73,20 @@ get_ordinance_ancestors <- function(product_id, node_id) { tidyr::unnest_longer(Ancestors) %>% janitor::clean_names() %>% dplyr::select(id, ancestors, ancestors_id) - - return(result) } #' Get the children of a given node in an ordinance's Table of Contents #' @description Corresponds to, for example: https://api.municode.com/ordinancesToc/children?productId=12429&nodeId=2023 -#' @param product_id A unique identifier for a product #' @param node_id A unique identifier for a node within the specified product (ordinance) +#' @param product_id A unique identifier for a product #' @export -get_ordinance_children <- function(product_id, node_id) { - result <- - build_endpoint( - domain = "ordinancesToc", - subdomain = "children", - parameters = c(nodeId = node_id, productId = product_id)) %>% +get_ordinance_children <- function(node_id, product_id) { + build_endpoint( + domain = "ordinancesToc", + subdomain = "children", + parameters = c(nodeId = node_id, productId = product_id)) %>% get_endpoint() %>% tibble::enframe() %>% tidyr::unnest_wider(value) %>% tidyr::unnest_wider(Data) - - return(result) } diff --git a/R/products.R b/R/products.R index 3ae998e..8508267 100644 --- a/R/products.R +++ b/R/products.R @@ -11,16 +11,31 @@ #' get_product_metadata(client_id = 12053, product_name = "Code of Ordinances") #' } get_product_metadata <- function(client_id, product_name) { - product_name <- product_name %>% - stringr::str_replace_all(" ", "+") %>% - stringr::str_to_lower() + ## The API matches product names case-insensitively. Spaces are left intact; + ## build_endpoint() percent-encodes query values, so manual substitution is + ## both unnecessary and wrong (it would encode the literal "+"). + product_name <- stringr::str_to_lower(product_name) - result <- + response <- build_endpoint( domain = "Products", subdomain = "name", parameters = c(clientId = client_id, productName = product_name)) %>% - get_endpoint() %>% + get_endpoint() + + ## The API now wraps this endpoint's payload in an envelope: + ## {IsSuccess, IsError, Message, Model}. The product fields live in Model. + if (!is.null(response$Model) || "IsSuccess" %in% names(response)) { + if (isTRUE(response$IsError) || is.null(response$Model)) { + stop( + "Municode API returned an error for client_id ", client_id, + ", product_name '", product_name, "': ", response$Message) + } + response <- response$Model + } + + result <- + response %>% tibble::enframe() %>% tidyr::pivot_wider() %>% tidyr::unnest_wider(ContentType) %>% @@ -32,7 +47,10 @@ get_product_metadata <- function(client_id, product_name) { dplyr::matches("client")) %>% dplyr::select(-c(client_pop_range_id, client_classification_id, client_show_advance_sheet, dplyr::matches("client_library|meetings"))) %>% dplyr::rename_with(~ stringr::str_replace_all(.x, "client_client", "client")) %>% - dplyr::select(dplyr::where(~ !is.list(.x) || (is.list(.x) && length(.x[[1]]) == 1))) + ## Keep non-list columns, plus any list column whose every element is a + ## single scalar (checking all rows, not just the first, avoids ragged + ## columns slipping through and misaligning on a later unlist). + dplyr::select(dplyr::where(~ !is.list(.x) || all(lengths(.x) == 1))) - return(result) + result } diff --git a/R/states.R b/R/states.R index cc2bff7..d641227 100644 --- a/R/states.R +++ b/R/states.R @@ -5,13 +5,10 @@ #' @returns A dataframe with three columns: `state_id`, `state_name`, `state_abbreviation.` #' @export get_states <- function() { - result <- - build_endpoint("States") %>% + build_endpoint("States") %>% get_endpoint() %>% transform_list_response() %>% dplyr::select(-name) - - return(result) } #' Look up state metadata given a state abbreviation @@ -21,16 +18,13 @@ get_states <- function() { #' @returns A dataframe with three columns: `state_id`, `state_name`, `state_abbreviation.` #' @export get_state_by_abbreviation <- function(state_abbreviation) { - result <- - build_endpoint( - domain = "States", - subdomain = "abbr", - parameters = c(stateAbbr = state_abbreviation)) %>% + build_endpoint( + domain = "States", + subdomain = "abbr", + parameters = c(stateAbbr = state_abbreviation)) %>% get_endpoint() %>% tibble::enframe() %>% dplyr::mutate(value = unlist(value)) %>% tidyr::pivot_wider() %>% janitor::clean_names() - - return(result) } diff --git a/R/utils-api.R b/R/utils-api.R index ba94cb4..d7d2a6a 100644 --- a/R/utils-api.R +++ b/R/utils-api.R @@ -1,7 +1,3 @@ -#' @title API utility functions for municode.com -#' @description Internal functions for building endpoints and making API calls -#' @keywords internal - #' Build an API endpoint URL #' @param domain The first component of the API endpoint after base api.municode.com/ #' @param subdomain The second component of the API endpoint @@ -9,34 +5,26 @@ #' @return Complete API endpoint URL #' @keywords internal build_endpoint <- function(domain, subdomain = NULL, parameters = NULL) { - # Build base URL - base_url <- "https://api.municode.com/" - - # Build path path_parts <- c(domain, subdomain) - path <- paste(stats::na.omit(path_parts), collapse = "/") + path_parts <- path_parts[!is.na(path_parts)] - # Combine base and path - url <- paste0(base_url, path) + ## httr2 percent-encodes path segments and query values, so callers never + ## need to pre-encode (e.g. client or product names containing spaces). + ## rlang::exec() splices the variable-length path parts and parameters in, + ## since req_url_path_append() does not itself support dynamic dots. + request <- rlang::exec( + httr2::req_url_path_append, + httr2::request("https://api.municode.com/"), + !!!as.list(path_parts)) - # Add query parameters if provided if (!is.null(parameters)) { - # Remove NULL parameters - parameters <- parameters[!sapply(parameters, is.null)] parameters <- parameters[!is.na(parameters)] - if (length(parameters) > 0) { - params_str <- paste( - names(parameters), - sapply(parameters, as.character), - sep = "=", - collapse = "&" - ) - url <- paste0(url, "?", params_str) + request <- rlang::exec(httr2::req_url_query, request, !!!as.list(parameters)) } } - return(url) + request$url } #' Send a GET request to a municode API endpoint @@ -45,39 +33,42 @@ build_endpoint <- function(domain, subdomain = NULL, parameters = NULL) { #' @return Parsed JSON response #' @keywords internal get_endpoint <- function(endpoint, max_retries = 3) { - tryCatch({ - result <- endpoint %>% + fail <- function(e) { + stop(sprintf( + "Failed to fetch from Municode API.\nEndpoint: %s\nError: %s", + endpoint, conditionMessage(e) + ), call. = FALSE) + } + + ## Connection-level failures (DNS, timeout, refused) and non-JSON bodies are + ## wrapped via fail(); a genuine HTTP status error is reported separately so + ## the status code is surfaced rather than buried in a generic message. + response <- tryCatch( + endpoint %>% httr2::request() %>% - httr2::req_retry(max_tries = max_retries) %>% + httr2::req_timeout(30) %>% + httr2::req_retry( + max_tries = max_retries, + is_transient = function(resp) httr2::resp_status(resp) %in% c(429L, 500L, 502L, 503L, 504L)) %>% httr2::req_error(is_error = function(resp) FALSE) %>% - httr2::req_perform() - - # Check for HTTP errors - if (httr2::resp_is_error(result)) { - status <- httr2::resp_status(result) - stop(sprintf( - "Municode API request failed (HTTP %d).\nEndpoint: %s", - status, endpoint - ), call. = FALSE) - } - - # Parse JSON response - parsed <- httr2::resp_body_json(result) + httr2::req_perform(), + error = fail) - # Validate response - if (is.null(parsed)) { - warning(sprintf( - "API returned NULL response for endpoint: %s", - endpoint - ), call. = FALSE) - } + if (httr2::resp_is_error(response)) { + stop(sprintf( + "Municode API request failed (HTTP %d).\nEndpoint: %s", + httr2::resp_status(response), endpoint + ), call. = FALSE) + } - return(parsed) + parsed <- tryCatch(httr2::resp_body_json(response), error = fail) - }, error = function(e) { - stop(sprintf( - "Failed to fetch from Municode API.\nEndpoint: %s\nError: %s", - endpoint, conditionMessage(e) + if (is.null(parsed)) { + warning(sprintf( + "Municode API returned an empty response for endpoint: %s", + endpoint ), call. = FALSE) - }) + } + + parsed } diff --git a/R/utils-transform.R b/R/utils-transform.R index ed127c0..d8398a9 100644 --- a/R/utils-transform.R +++ b/R/utils-transform.R @@ -1,7 +1,3 @@ -#' @title Common data transformation helpers for municode API responses -#' @description Internal functions to standardize data processing across the package -#' @keywords internal - #' Transform a simple list response to a tidy dataframe #' @param response Raw API response (list) #' @param primary_key Name of the primary key in the response (default "value") @@ -33,17 +29,18 @@ transform_nested_client <- function(df) { tidyr::unnest_wider(Client, names_sep = "_") %>% tidyr::unnest_wider(Client_State) - # Try to unlist list columns, but don't fail if some can't be unlisted - tryCatch({ - result <- result %>% - dplyr::mutate(dplyr::across(dplyr::where(is.list), unlist)) - }, error = function(e) { - # If unlisting fails, just select out any remaining list columns - result <- result %>% - dplyr::select(dplyr::where(~ !is.list(.x))) - }) - - return(result) + ## Flatten list columns: NULL elements become NA so a single missing value + ## doesn't force the whole column (or, previously, every list column) to be + ## dropped. Columns whose elements are all scalars (after NULL -> NA) are + ## unlisted; only genuinely ragged columns are dropped. + result %>% + dplyr::mutate(dplyr::across( + dplyr::where(is.list), + ~ purrr::map(.x, \(element) if (is.null(element)) NA else element))) %>% + dplyr::mutate(dplyr::across( + dplyr::where(~ is.list(.x) && all(lengths(.x) == 1) && all(purrr::map_lgl(.x, is.atomic))), + unlist)) %>% + dplyr::select(dplyr::where(~ !is.list(.x))) } #' Clean HTML content from ordinance text @@ -54,7 +51,7 @@ clean_html_content <- function(content) { content %>% stringr::str_replace_all(c( "<.*?>" = "", # Remove HTML tags (non-greedy) - "\\\n" = "", # Remove newlines + "\\n" = "", # Remove newlines "\\ " = " ", # Replace   with space "\\s+" = " " # Collapse multiple spaces )) diff --git a/R/workflows.R b/R/workflows.R index 5f92b33..d4ea0dd 100644 --- a/R/workflows.R +++ b/R/workflows.R @@ -1,5 +1,42 @@ #' @title Workflow helper functions for common multi-step operations +#' Resolve a client_id from a state and client name, erroring if not found +#' @param state_abbreviation Two-character state code +#' @param client_name Name of the municipality +#' @return A single client_id +#' @keywords internal +resolve_client_id <- function(state_abbreviation, client_name) { + client_id <- get_client_metadata(state_abbreviation, client_name)$client_id + + if (is.null(client_id) || length(client_id) == 0) { + stop(sprintf( + "Could not find client '%s' in state '%s'", + client_name, state_abbreviation + ), call. = FALSE) + } + + client_id +} + +#' Resolve a product_id from a client_id and product name, erroring if not found +#' @param client_id A unique identifier for a client +#' @param product_name Name of the product +#' @param client_name Name of the municipality, used only for the error message +#' @return A single product_id +#' @keywords internal +resolve_product_id <- function(client_id, product_name, client_name) { + product_id <- get_product_metadata(client_id, product_name)$product_id + + if (is.null(product_id) || length(product_id) == 0) { + stop(sprintf( + "Could not find product '%s' for client '%s'", + product_name, client_name + ), call. = FALSE) + } + + product_id +} + #' Get ordinance section content in one step #' @description Convenience function that chains together multiple API calls to retrieve #' ordinance section content given just the jurisdiction, product name, and node ID. @@ -21,32 +58,10 @@ #' ) #' } get_ordinance_section <- function(state_abbreviation, client_name, product_name, node_id) { - # Step 1: Get client metadata - client_meta <- get_client_metadata(state_abbreviation, client_name) - client_id <- client_meta$client_id - - if (is.null(client_id) || length(client_id) == 0) { - stop(sprintf( - "Could not find client '%s' in state '%s'", - client_name, state_abbreviation - ), call. = FALSE) - } - - # Step 2: Get product metadata - product_meta <- get_product_metadata(client_id, product_name) - product_id <- product_meta$product_id - - if (is.null(product_id) || length(product_id) == 0) { - stop(sprintf( - "Could not find product '%s' for client '%s'", - product_name, client_name - ), call. = FALSE) - } - - # Step 3: Get content - content <- get_section_text(node_id, product_id) + client_id <- resolve_client_id(state_abbreviation, client_name) + product_id <- resolve_product_id(client_id, product_name, client_name) - return(content) + get_section_text(node_id, product_id) } #' Get full table of contents for an ordinance in one step @@ -68,31 +83,10 @@ get_ordinance_section <- function(state_abbreviation, client_name, product_name, #' ) #' } get_ordinance_toc <- function(state_abbreviation, client_name, product_name) { - # Step 1: Get client metadata - client_meta <- get_client_metadata(state_abbreviation, client_name) - client_id <- client_meta$client_id - - if (is.null(client_id) || length(client_id) == 0) { - stop(sprintf( - "Could not find client '%s' in state '%s'", - client_name, state_abbreviation - ), call. = FALSE) - } - - # Step 2: Get product metadata - product_meta <- get_product_metadata(client_id, product_name) - product_id <- product_meta$product_id - - if (is.null(product_id) || length(product_id) == 0) { - stop(sprintf( - "Could not find product '%s' for client '%s'", - product_name, client_name - ), call. = FALSE) - } + client_id <- resolve_client_id(state_abbreviation, client_name) + product_id <- resolve_product_id(client_id, product_name, client_name) - # Step 3: Get latest job - job <- get_current_version(product_id) - job_id <- job$id + job_id <- get_current_version(product_id)$id if (is.null(job_id) || length(job_id) == 0) { stop(sprintf( @@ -101,10 +95,7 @@ get_ordinance_toc <- function(state_abbreviation, client_name, product_name) { ), call. = FALSE) } - # Step 4: Get table of contents - toc <- get_codes_toc(job_id, product_id) - - return(toc) + get_codes_toc(job_id, product_id) } #' Get all products for a jurisdiction in one step @@ -124,19 +115,7 @@ get_ordinance_toc <- function(state_abbreviation, client_name, product_name) { #' ) #' } get_jurisdiction_products <- function(state_abbreviation, client_name) { - # Get client metadata - client_meta <- get_client_metadata(state_abbreviation, client_name) - client_id <- client_meta$client_id - - if (is.null(client_id) || length(client_id) == 0) { - stop(sprintf( - "Could not find client '%s' in state '%s'", - client_name, state_abbreviation - ), call. = FALSE) - } - - # Get products - products <- get_client_products(client_id) + client_id <- resolve_client_id(state_abbreviation, client_name) - return(products) + get_client_products(client_id) } diff --git a/README.Rmd b/README.Rmd index f6ed171..c5029ff 100644 --- a/README.Rmd +++ b/README.Rmd @@ -46,6 +46,27 @@ set_urbn_defaults() options(scipen = 999999) ``` +```{r} +https://library.municode.com/api/ClientContent/17313 +https://library.municode.com/api/States/abbr?stateAbbr=or +https://library.municode.com/api/ClientContent/17313 +https://library.municode.com/api/Products/name?clientId=17313&productName=code+of+ordinances +https://library.municode.com/api/Jobs/latest/15441 +https://library.municode.com/api/codesToc/breadcrumb?jobId=494092&nodeId=15441&productId=15441 +https://library.municode.com/api/codesToc?jobId=494092&productId=15441 +https://library.municode.com/api/CodesContent?jobId=494092&productId=15441 +https://library.municode.com/localapi/Codebank/GetVersions/4052 +https://library.municode.com/api/Jobs/15441/488221 +https://library.municode.com/api/CodesChangedDocs?jobId=488221&productId=15441 +https://library.municode.com/api/CodesChangedDocs?jobId=494092&productId=15441 +https://library.municode.com/api/PublicationPdfDownload/4052 +https://library.municode.com/api/CoreContent/Ordinances?productId=15441 +https://library.municode.com/api/codesToc/fullTree?jobId=494092&nodeId=15441&productId=15441 +https://library.municode.com/api/DocActions/saveAsDocx/15441 +https://library.municode.com/api/codesContent?jobId=494092&nodeId=TITXUNDECO_UDC_CH110GEZOPR_S110.050RESARECORE&pastJobId=491712&productId=15441&showChanges=true +``` + + ## Quick Start Get started with municoder in just a few lines: diff --git a/man/build_endpoint.Rd b/man/build_endpoint.Rd index 267a6ae..0fbb6d8 100644 --- a/man/build_endpoint.Rd +++ b/man/build_endpoint.Rd @@ -2,7 +2,7 @@ % Please edit documentation in R/utils-api.R \name{build_endpoint} \alias{build_endpoint} -\title{API utility functions for municode.com} +\title{Build an API endpoint URL} \usage{ build_endpoint(domain, subdomain = NULL, parameters = NULL) } @@ -17,11 +17,6 @@ build_endpoint(domain, subdomain = NULL, parameters = NULL) Complete API endpoint URL } \description{ -Internal functions for building endpoints and making API calls +Build an API endpoint URL } -\keyword{API} -\keyword{Build} -\keyword{URL} -\keyword{an} -\keyword{endpoint} \keyword{internal} diff --git a/man/extract_section_docs.Rd b/man/extract_section_docs.Rd new file mode 100644 index 0000000..70f2dfa --- /dev/null +++ b/man/extract_section_docs.Rd @@ -0,0 +1,18 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/codes.R +\name{extract_section_docs} +\alias{extract_section_docs} +\title{Extract the Docs payload (id, title, content) from a CodesContent response} +\usage{ +extract_section_docs(result) +} +\arguments{ +\item{result}{A reshaped CodesContent response from \code{fetch_section_result()}} +} +\value{ +A tibble with cleaned \code{id}, \code{title}, and \code{content} columns +} +\description{ +Extract the Docs payload (id, title, content) from a CodesContent response +} +\keyword{internal} diff --git a/man/fetch_section_result.Rd b/man/fetch_section_result.Rd new file mode 100644 index 0000000..137d0ae --- /dev/null +++ b/man/fetch_section_result.Rd @@ -0,0 +1,36 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/codes.R +\name{fetch_section_result} +\alias{fetch_section_result} +\title{Code-related API functions (current ordinance versions) +Fetch and reshape the raw CodesContent response for a node} +\usage{ +fetch_section_result( + node_id, + product_id, + job_id = NULL, + past_job_id = NULL, + show_changes = FALSE +) +} +\arguments{ +\item{node_id}{A unique identifier for a node within the specified product} + +\item{product_id}{A unique identifier for a product} + +\item{job_id}{Optional job identifier; required when \code{show_changes = TRUE}} + +\item{past_job_id}{Optional prior job to diff against; required when +\code{show_changes = TRUE}} + +\item{show_changes}{If \code{TRUE}, request redline markup comparing \code{job_id} +against \code{past_job_id}} +} +\value{ +A one-row tibble with one column per top-level response field +} +\description{ +Code-related API functions (current ordinance versions) +Fetch and reshape the raw CodesContent response for a node +} +\keyword{internal} diff --git a/man/flatten_codes_tree.Rd b/man/flatten_codes_tree.Rd new file mode 100644 index 0000000..f9ad841 --- /dev/null +++ b/man/flatten_codes_tree.Rd @@ -0,0 +1,18 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/codes.R +\name{flatten_codes_tree} +\alias{flatten_codes_tree} +\title{Recursively flatten a codesToc tree node into a tidy dataframe} +\usage{ +flatten_codes_tree(node) +} +\arguments{ +\item{node}{A node list from the \code{codesToc/fullTree} response} +} +\value{ +A dataframe with one row per node in the subtree rooted at \code{node} +} +\description{ +Recursively flatten a codesToc tree node into a tidy dataframe +} +\keyword{internal} diff --git a/man/get_changed_sections.Rd b/man/get_changed_sections.Rd new file mode 100644 index 0000000..454cf08 --- /dev/null +++ b/man/get_changed_sections.Rd @@ -0,0 +1,32 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/jobs.R +\name{get_changed_sections} +\alias{get_changed_sections} +\title{List the sections that changed in a given version of an ordinance} +\usage{ +get_changed_sections(job_id, product_id) +} +\arguments{ +\item{job_id}{A unique identifier for a job} + +\item{product_id}{A unique identifier for a product} +} +\value{ +A dataframe with one row per changed section and the columns +\code{node_id}, \code{title}, \code{compare_status} (an undocumented change-type code), +and \code{ancestor_path} (the node's breadcrumb, titles joined by \code{" > "}). +Returns a zero-row dataframe when the version introduced no changes. +} +\description{ +Corresponds to, for example: https://api.municode.com/CodesChangedDocs?jobId=494092&productId=15441 +Returns the set of sections (nodes) that were added or amended in the +specified job relative to the preceding version. Useful for tracking +ordinance changes over time; pair with \code{\link[=get_version_history]{get_version_history()}} to walk +versions and \code{\link[=get_section_html]{get_section_html()}} (with \code{show_changes = TRUE}) to retrieve +the redline for each changed node. +} +\examples{ +\dontrun{ +get_changed_sections(job_id = 494092, product_id = 15441) +} +} diff --git a/man/get_codes_tree.Rd b/man/get_codes_tree.Rd new file mode 100644 index 0000000..62b4ed7 --- /dev/null +++ b/man/get_codes_tree.Rd @@ -0,0 +1,34 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/codes.R +\name{get_codes_tree} +\alias{get_codes_tree} +\title{Get the full table of contents tree for an ordinance in a single request} +\usage{ +get_codes_tree(job_id, node_id, product_id) +} +\arguments{ +\item{job_id}{A unique identifier for a job} + +\item{node_id}{A node to use as the root of the returned tree. Pass the +\code{product_id} to retrieve the whole code.} + +\item{product_id}{A unique identifier for a product} +} +\value{ +A dataframe with one row per node and the columns \code{id}, \code{heading}, +\code{node_depth}, \code{parent_id}, \code{doc_order_id}, \code{has_children}, \code{node_key}, +\code{is_updated}, \code{is_amended}, \code{has_amended_descendant}, \code{compare_status}, +and \code{doc_type}. +} +\description{ +Corresponds to, for example: https://api.municode.com/codesToc/fullTree?jobId=494092&nodeId=15441&productId=15441 +Unlike \code{\link[=get_section_children]{get_section_children()}}, which returns one level of the hierarchy +per call, this returns the entire nested tree beneath \code{node_id} in a single +request and flattens it to a tidy dataframe (one row per node). Per-node +amendment flags are included. +} +\examples{ +\dontrun{ +get_codes_tree(job_id = 494092, node_id = 15441, product_id = 15441) +} +} diff --git a/man/get_ordinance_ancestors.Rd b/man/get_ordinance_ancestors.Rd index 43dfb3f..2c3db84 100644 --- a/man/get_ordinance_ancestors.Rd +++ b/man/get_ordinance_ancestors.Rd @@ -4,12 +4,12 @@ \alias{get_ordinance_ancestors} \title{Get the ancestors of a given node in an ordinance's Table of Contents} \usage{ -get_ordinance_ancestors(product_id, node_id) +get_ordinance_ancestors(node_id, product_id) } \arguments{ -\item{product_id}{A unique identifier for a product} - \item{node_id}{A unique identifier for a node within the specified product (ordinance)} + +\item{product_id}{A unique identifier for a product} } \description{ Corresponds to, for example: https://api.municode.com/ordinancesToc/breadcrumb?nodeId=2023&productId=12429 diff --git a/man/get_ordinance_children.Rd b/man/get_ordinance_children.Rd index 8d11aa7..bd87e47 100644 --- a/man/get_ordinance_children.Rd +++ b/man/get_ordinance_children.Rd @@ -4,12 +4,12 @@ \alias{get_ordinance_children} \title{Get the children of a given node in an ordinance's Table of Contents} \usage{ -get_ordinance_children(product_id, node_id) +get_ordinance_children(node_id, product_id) } \arguments{ -\item{product_id}{A unique identifier for a product} - \item{node_id}{A unique identifier for a node within the specified product (ordinance)} + +\item{product_id}{A unique identifier for a product} } \description{ Corresponds to, for example: https://api.municode.com/ordinancesToc/children?productId=12429&nodeId=2023 diff --git a/man/get_ordinance_section.Rd b/man/get_ordinance_section.Rd index 16ae208..10bbee6 100644 --- a/man/get_ordinance_section.Rd +++ b/man/get_ordinance_section.Rd @@ -2,8 +2,7 @@ % Please edit documentation in R/workflows.R \name{get_ordinance_section} \alias{get_ordinance_section} -\title{Workflow helper functions for common multi-step operations -Get ordinance section content in one step} +\title{Get ordinance section content in one step} \usage{ get_ordinance_section(state_abbreviation, client_name, product_name, node_id) } diff --git a/man/get_ordinances_toc.Rd b/man/get_ordinances_toc.Rd index bda7565..1af332d 100644 --- a/man/get_ordinances_toc.Rd +++ b/man/get_ordinances_toc.Rd @@ -4,12 +4,12 @@ \alias{get_ordinances_toc} \title{Obtain metadata about a given ordinance over time} \usage{ -get_ordinances_toc(product_id, node_id = NA) +get_ordinances_toc(node_id = NULL, product_id) } \arguments{ -\item{product_id}{A unique identifier for a product} - \item{node_id}{A unique identifier for a node within the specified product (ordinance)} + +\item{product_id}{A unique identifier for a product} } \description{ Corresponds to, for example: https://api.municode.com/ordinancesToc?nodeId=2023&productId=12429 diff --git a/man/get_section_html.Rd b/man/get_section_html.Rd index 2bccbef..35c819e 100644 --- a/man/get_section_html.Rd +++ b/man/get_section_html.Rd @@ -4,12 +4,27 @@ \alias{get_section_html} \title{Get raw HTML content and markdown for a given node within a given ordinance} \usage{ -get_section_html(node_id = NULL, product_id) +get_section_html( + node_id = NULL, + product_id, + job_id = NULL, + past_job_id = NULL, + show_changes = FALSE +) } \arguments{ \item{node_id}{A unique identifier for a node within the specified product (ordinance)} \item{product_id}{A unique identifier for a product} + +\item{job_id}{Optional job identifier; required when \code{show_changes = TRUE}} + +\item{past_job_id}{Optional prior job identifier to diff against; required +when \code{show_changes = TRUE}} + +\item{show_changes}{If \code{TRUE}, request a redline comparison of \code{job_id} +against \code{past_job_id}. The insertion/deletion markup is preserved in the +returned \code{content_html} column.} } \value{ A dataframe with the raw HTML content and metadata for the specified node. diff --git a/man/get_section_text.Rd b/man/get_section_text.Rd index 4ec4bd6..73d52fa 100644 --- a/man/get_section_text.Rd +++ b/man/get_section_text.Rd @@ -2,15 +2,30 @@ % Please edit documentation in R/codes.R \name{get_section_text} \alias{get_section_text} -\title{Code-related API functions (current ordinance versions) -Get text and metadata for a given node within a given ordinance} +\title{Get text and metadata for a given node within a given ordinance} \usage{ -get_section_text(node_id = NULL, product_id) +get_section_text( + node_id = NULL, + product_id, + job_id = NULL, + past_job_id = NULL, + show_changes = FALSE +) } \arguments{ \item{node_id}{A unique identifier for a node within the specified product (ordinance)} \item{product_id}{A unique identifier for a product} + +\item{job_id}{Optional job identifier; required when \code{show_changes = TRUE}} + +\item{past_job_id}{Optional prior job identifier to diff against; required +when \code{show_changes = TRUE}} + +\item{show_changes}{If \code{TRUE}, request a redline comparison of \code{job_id} +against \code{past_job_id}. Note that \code{get_section_text()} strips HTML, which +discards the insertion/deletion markup; use \code{\link[=get_section_html]{get_section_html()}} to retain +the redline in the \code{content_html} column.} } \value{ A dataframe with the content and metadata pertaining to a specific node within an ordinance diff --git a/man/list_ordinances.Rd b/man/list_ordinances.Rd index 3b8de9d..fd9010a 100644 --- a/man/list_ordinances.Rd +++ b/man/list_ordinances.Rd @@ -5,15 +5,19 @@ \title{Ordinance-related API functions (historical ordinance versions) Return information about all ordinances of a given type for a given year} \usage{ -list_ordinances(product_id, node_id = NULL) +list_ordinances(node_id = NULL, product_id) } \arguments{ -\item{product_id}{A unique code identifying a product (e.g., a zoning ordinance)} +\item{node_id}{The year for which ordinances are requested. Optional; when +omitted the response is the top-level index rather than a list of ordinances.} -\item{node_id}{The year for which ordinances are requested} +\item{product_id}{A unique code identifying a product (e.g., a zoning ordinance)} } \value{ -A dataframe comprising ordinances +A dataframe comprising ordinances. The shape depends on \code{node_id}: +when the response contains nested list elements (typically when \code{node_id} +is supplied) each ordinance becomes a row; otherwise the top-level fields +are pivoted into a single row. } \description{ Corresponds to, for example: https://api.municode.com/CoreContent/Ordinances?nodeId=2023&productId=12429 diff --git a/man/municoder-package.Rd b/man/municoder-package.Rd index bcf8e8e..147fb0f 100644 --- a/man/municoder-package.Rd +++ b/man/municoder-package.Rd @@ -6,7 +6,7 @@ \alias{municoder-package} \title{municoder: Interface to the municode.com API} \description{ -Access and programmatically navigated ordinances and associated metadata relating to regulatory documents hosted on municode.com. +Access and programmatically navigate ordinances and associated metadata relating to regulatory documents hosted on municode.com. } \seealso{ Useful links: @@ -18,5 +18,10 @@ Useful links: \author{ \strong{Maintainer}: Will Curran-Groome \email{wcurrangroome@gmail.com} +Authors: +\itemize{ + \item Will Curran-Groome \email{wcurrangroome@gmail.com} +} + } \keyword{internal} diff --git a/man/resolve_client_id.Rd b/man/resolve_client_id.Rd new file mode 100644 index 0000000..646b3f4 --- /dev/null +++ b/man/resolve_client_id.Rd @@ -0,0 +1,22 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/workflows.R +\name{resolve_client_id} +\alias{resolve_client_id} +\title{Workflow helper functions for common multi-step operations +Resolve a client_id from a state and client name, erroring if not found} +\usage{ +resolve_client_id(state_abbreviation, client_name) +} +\arguments{ +\item{state_abbreviation}{Two-character state code} + +\item{client_name}{Name of the municipality} +} +\value{ +A single client_id +} +\description{ +Workflow helper functions for common multi-step operations +Resolve a client_id from a state and client name, erroring if not found +} +\keyword{internal} diff --git a/man/resolve_product_id.Rd b/man/resolve_product_id.Rd new file mode 100644 index 0000000..e5f5f13 --- /dev/null +++ b/man/resolve_product_id.Rd @@ -0,0 +1,22 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/workflows.R +\name{resolve_product_id} +\alias{resolve_product_id} +\title{Resolve a product_id from a client_id and product name, erroring if not found} +\usage{ +resolve_product_id(client_id, product_name, client_name) +} +\arguments{ +\item{client_id}{A unique identifier for a client} + +\item{product_name}{Name of the product} + +\item{client_name}{Name of the municipality, used only for the error message} +} +\value{ +A single product_id +} +\description{ +Resolve a product_id from a client_id and product name, erroring if not found +} +\keyword{internal} diff --git a/man/transform_list_response.Rd b/man/transform_list_response.Rd index d348e9c..1bd8b7f 100644 --- a/man/transform_list_response.Rd +++ b/man/transform_list_response.Rd @@ -2,7 +2,7 @@ % Please edit documentation in R/utils-transform.R \name{transform_list_response} \alias{transform_list_response} -\title{Common data transformation helpers for municode API responses} +\title{Transform a simple list response to a tidy dataframe} \usage{ transform_list_response(response, primary_key = "value") } @@ -15,14 +15,6 @@ transform_list_response(response, primary_key = "value") A tidy dataframe } \description{ -Internal functions to standardize data processing across the package +Transform a simple list response to a tidy dataframe } -\keyword{Transform} -\keyword{a} -\keyword{dataframe} \keyword{internal} -\keyword{list} -\keyword{response} -\keyword{simple} -\keyword{tidy} -\keyword{to} diff --git a/tests/testthat/test-clients.R b/tests/testthat/test-clients.R index c04c933..f482c70 100644 --- a/tests/testthat/test-clients.R +++ b/tests/testthat/test-clients.R @@ -51,7 +51,7 @@ test_that("get_client_metadata handles invalid client names", { expect_error( get_client_metadata("VA", "NonexistentCity12345"), - "Failed to fetch|Can't compute column|NULL" + "Failed to fetch|Municode API request failed|Can't compute column|NULL" ) }) @@ -74,6 +74,6 @@ test_that("get_client_products handles invalid client_id", { expect_error( get_client_products(999999999), - "Failed to fetch|Can't compute column|Can't select columns|doesn't exist" + "Failed to fetch|Municode API request failed|Can't compute column|Can't select columns|doesn't exist" ) }) diff --git a/tests/testthat/test-codes.R b/tests/testthat/test-codes.R index 988b451..d263e5e 100644 --- a/tests/testthat/test-codes.R +++ b/tests/testthat/test-codes.R @@ -92,6 +92,56 @@ test_that("get_section_text handles invalid node_id", { expect_error( get_section_text(node_id = "INVALID123", product_id = 12429), - "Failed to fetch|Can't compute column|NULL" + "Failed to fetch|Municode API request failed|Can't compute column|NULL" ) }) + +test_that("flatten_codes_tree flattens a nested tree to one row per node", { + tree <- list( + Id = "ROOT", Heading = "Root", NodeDepth = -1L, HasChildren = TRUE, + ParentId = NULL, DocOrderId = 0L, + Data = list(NodeKey = "k0", IsUpdated = FALSE, IsAmended = FALSE, + HasAmendedDescendant = TRUE, CompareStatus = 0L, DocType = 1L), + Children = list( + list(Id = "A", Heading = "Child A", NodeDepth = 0L, HasChildren = FALSE, + ParentId = "ROOT", DocOrderId = 1L, + Data = list(NodeKey = "k1", IsUpdated = FALSE, IsAmended = TRUE, + HasAmendedDescendant = FALSE, CompareStatus = 2L, DocType = 1L), + Children = list()), + list(Id = "B", Heading = "Child B", NodeDepth = 0L, HasChildren = FALSE, + ParentId = "ROOT", DocOrderId = 2L, + Data = list(NodeKey = "k2", IsUpdated = FALSE, IsAmended = FALSE, + HasAmendedDescendant = FALSE, CompareStatus = 0L, DocType = 1L), + Children = list()))) + + result <- municoder:::flatten_codes_tree(tree) + + expect_s3_class(result, "data.frame") + expect_equal(nrow(result), 3) + # Root first, then children in order + expect_equal(result$id, c("ROOT", "A", "B")) + # NULL ParentId on the root becomes NA, not an error + expect_equal(result$parent_id, c(NA, "ROOT", "ROOT")) + expect_true(result$is_amended[result$id == "A"]) + # Everything is unnested -- no residual list columns + expect_false(any(vapply(result, is.list, logical(1)))) +}) + +test_that("get_section_html show_changes requires job_id and past_job_id", { + expect_error( + get_section_html(node_id = "SUHITA", product_id = 15441, show_changes = TRUE), + "requires both" + ) +}) + +test_that("get_codes_tree returns a flattened tree with one row per node", { + skip_on_cran() + skip_if_offline() + + result <- get_codes_tree(job_id = 494092, node_id = 15441, product_id = 15441) + + expect_s3_class(result, "data.frame") + expect_gt(nrow(result), 1) + expect_true(all(c("id", "heading", "node_depth", "parent_id", "is_amended") %in% names(result))) + expect_false(any(vapply(result, is.list, logical(1)))) +}) diff --git a/tests/testthat/test-jobs.R b/tests/testthat/test-jobs.R index deb031a..58726ba 100644 --- a/tests/testthat/test-jobs.R +++ b/tests/testthat/test-jobs.R @@ -42,7 +42,7 @@ test_that("get_current_version handles invalid product_id", { expect_error( get_current_version(999999999), - "Failed to fetch|Can't compute column" + "Failed to fetch|Municode API request failed|Can't compute column" ) }) @@ -63,7 +63,7 @@ test_that("get_version_history handles invalid product_id", { expect_error( get_version_history(999999999), - "Failed to fetch|Can't compute column" + "Failed to fetch|Municode API request failed|Can't compute column" ) }) @@ -76,3 +76,24 @@ test_that("get_version_history returns historical versions", { # A product should have at least 1 job (current version) expect_gte(nrow(result), 1) }) + +test_that("get_changed_sections returns changed sections for a version", { + skip_on_cran() + skip_if_offline() + + result <- get_changed_sections(job_id = 494092, product_id = 15441) + + expect_s3_class(result, "data.frame") + expect_true(all(c("node_id", "title", "compare_status", "ancestor_path") %in% names(result))) +}) + +test_that("get_changed_sections returns a typed zero-row frame when nothing changed", { + skip_on_cran() + skip_if_offline() + + # Job 488221 of product 15441 introduced no tracked changes + result <- get_changed_sections(job_id = 488221, product_id = 15441) + + expect_s3_class(result, "data.frame") + expect_equal(names(result), c("node_id", "title", "compare_status", "ancestor_path")) +}) diff --git a/tests/testthat/test-ordinances.R b/tests/testthat/test-ordinances.R index 34c7a59..1f7c206 100644 --- a/tests/testthat/test-ordinances.R +++ b/tests/testthat/test-ordinances.R @@ -60,7 +60,7 @@ test_that("get_ordinances functions handle invalid product_id", { expect_error( get_ordinances_toc(product_id = 999999999), - "Failed to fetch|Can't compute column" + "Failed to fetch|Municode API request failed|Can't compute column" ) }) @@ -70,6 +70,6 @@ test_that("list_ordinances handles invalid parameters", { expect_error( list_ordinances(product_id = 999999999, node_id = "INVALID"), - "Failed to fetch|Can't compute column" + "Failed to fetch|Municode API request failed|Can't compute column" ) }) diff --git a/tests/testthat/test-products.R b/tests/testthat/test-products.R index 529d341..cdbf253 100644 --- a/tests/testthat/test-products.R +++ b/tests/testthat/test-products.R @@ -11,11 +11,11 @@ test_that("get_product_metadata returns data for valid product", { expect_true("product_name" %in% names(result)) }) -test_that("get_product_metadata handles space conversion", { +test_that("get_product_metadata handles product names with spaces", { skip_on_cran() skip_if_offline() - # Spaces should be converted to + + # Spaces are percent-encoded by build_endpoint(), not pre-substituted result <- get_product_metadata(12053, "Code of Ordinances") expect_s3_class(result, "data.frame") @@ -51,7 +51,7 @@ test_that("get_product_metadata handles invalid product names", { expect_error( get_product_metadata(12053, "NonexistentProduct12345"), - "Failed to fetch|Can't compute column|NULL" + "Failed to fetch|Municode API request failed|Can't compute column|NULL" ) }) @@ -61,6 +61,6 @@ test_that("get_product_metadata handles invalid client_id", { expect_error( get_product_metadata(999999999, "Zoning"), - "Failed to fetch|Can't compute column" + "Failed to fetch|Municode API request failed|Can't compute column" ) }) diff --git a/tests/testthat/test-states.R b/tests/testthat/test-states.R index a06143f..870cf27 100644 --- a/tests/testthat/test-states.R +++ b/tests/testthat/test-states.R @@ -47,6 +47,6 @@ test_that("get_state_by_abbreviation handles invalid abbreviations", { expect_error( get_state_by_abbreviation("ZZ"), - "Failed to fetch|Can't compute column" + "Failed to fetch|Municode API request failed|Can't compute column" ) }) diff --git a/tests/testthat/test-utils-api.R b/tests/testthat/test-utils-api.R index de0e362..62fcdea 100644 --- a/tests/testthat/test-utils-api.R +++ b/tests/testthat/test-utils-api.R @@ -26,8 +26,8 @@ test_that("build_endpoint constructs URLs correctly", { # Test with multiple parameters result <- build_endpoint("Clients", subdomain = "name", parameters = c(stateAbbr = "VA", clientName = "Alexandria")) - expect_true(grepl("stateAbbr=VA", result)) - expect_true(grepl("clientName=Alexandria", result)) + expect_true(stringr::str_detect(result, "stateAbbr=VA")) + expect_true(stringr::str_detect(result, "clientName=Alexandria")) # Test with NULL parameters expect_equal( @@ -42,11 +42,15 @@ test_that("build_endpoint constructs URLs correctly", { ) }) -test_that("get_endpoint handles errors gracefully", { - # Test with invalid URL +test_that("get_endpoint surfaces HTTP errors with the status code", { + skip_on_cran() + skip_if_offline() + + # A reachable host returning a non-2xx status reports the HTTP status + # rather than being collapsed into the generic connection-failure message. expect_error( get_endpoint("https://api.municode.com/InvalidEndpoint"), - "Failed to fetch from Municode API" + "Municode API request failed \\(HTTP" ) }) diff --git a/tests/testthat/test-utils-transform.R b/tests/testthat/test-utils-transform.R index 356a6a4..37300dd 100644 --- a/tests/testthat/test-utils-transform.R +++ b/tests/testthat/test-utils-transform.R @@ -17,30 +17,30 @@ test_that("clean_html_content removes HTML tags", { html_text <- "

This is a test


With HTML" result <- municoder:::clean_html_content(html_text) - expect_false(grepl("<", result)) - expect_false(grepl(">", result)) - expect_true(grepl("This is a test", result)) + expect_false(stringr::str_detect(result, "<")) + expect_false(stringr::str_detect(result, ">")) + expect_true(stringr::str_detect(result, "This is a test")) }) test_that("clean_html_content removes   entities", { html_text <- "Test with spaces" result <- municoder:::clean_html_content(html_text) - expect_false(grepl(" ", result)) + expect_false(stringr::str_detect(result, " ")) }) test_that("clean_html_content collapses multiple spaces", { html_text <- "Test with spaces" result <- municoder:::clean_html_content(html_text) - expect_false(grepl(" ", result)) + expect_false(stringr::str_detect(result, " ")) }) test_that("clean_html_content removes newlines", { html_text <- "Test\nwith\nnewlines" result <- municoder:::clean_html_content(html_text) - expect_false(grepl("\n", result)) + expect_false(stringr::str_detect(result, "\n")) }) test_that("clean_client_columns removes specified columns", { @@ -66,3 +66,40 @@ test_that("clean_client_columns removes specified columns", { expect_true("client_name" %in% names(result)) expect_true("other_column" %in% names(result)) }) + +test_that("html_to_markdown converts headings, emphasis, and entities", { + expect_equal(municoder:::html_to_markdown("

Title

"), "# Title") + expect_equal(municoder:::html_to_markdown("

Sub

"), "## Sub") + expect_equal(municoder:::html_to_markdown("

Hello world

"), "Hello **world**") + expect_equal(municoder:::html_to_markdown("

a x

"), "a *x*") + expect_equal(municoder:::html_to_markdown("a & b"), "a & b") +}) + +test_that("html_to_markdown renders HTML tables as markdown tables", { + html <- "
AB
12
" + result <- municoder:::html_to_markdown(html) + + expect_true(stringr::str_detect(result, stringr::fixed("| A | B |"))) + expect_true(stringr::str_detect(result, stringr::fixed("| --- | --- |"))) + expect_true(stringr::str_detect(result, stringr::fixed("| 1 | 2 |"))) +}) + +test_that("convert_html_tables_to_markdown leaves table-free input untouched", { + expect_equal( + municoder:::convert_html_tables_to_markdown("no table here"), + "no table here" + ) +}) + +test_that("convert_html_tables_to_markdown is vectorized over its input", { + out <- municoder:::convert_html_tables_to_markdown(c("plain", "also plain")) + expect_length(out, 2) + expect_equal(out, c("plain", "also plain")) +}) + +test_that("clean_html_content strips tags, entities, and newlines together", { + expect_equal( + municoder:::clean_html_content("

Test text

\nLine"), + "Test textLine" + ) +}) diff --git a/tests/testthat/test-workflows.R b/tests/testthat/test-workflows.R index a69b9f6..c5c0e7e 100644 --- a/tests/testthat/test-workflows.R +++ b/tests/testthat/test-workflows.R @@ -58,7 +58,7 @@ test_that("get_ordinance_toc handles invalid product name", { expect_error( get_ordinance_toc("VA", "Alexandria", "NonexistentProduct"), - "Could not find product|Failed to fetch" + "Could not find product|Failed to fetch|Municode API request failed|Can't select columns" ) }) @@ -89,7 +89,7 @@ test_that("get_ordinance_section handles invalid node_id", { expect_error( get_ordinance_section("VA", "Alexandria", "Zoning", "INVALID123"), - "Failed to fetch|Can't compute column" + "Failed to fetch|Municode API request failed|Can't compute column" ) }) @@ -117,5 +117,5 @@ test_that("workflow functions provide informative errors", { error = function(e) conditionMessage(e) ) - expect_true(grepl("Could not find client|Failed to fetch|Unexpected content type", error_msg)) + expect_true(stringr::str_detect(error_msg, "Could not find client|Failed to fetch|Unexpected content type")) })