Data acquisition

Data acquisition#

This chapter pulls TTWA boundaries from the ONS Open Geography Portal and workplace-to-residence commuting flows from NOMIS. Outputs are cached as .rds files so later chapters can run without repeating slow API calls.

Previous: Introduction
Next: Data cleaning

suppressPackageStartupMessages({
  library(tidyverse)
  library(sf)
  library(here)
})

proj_dir <- here::here("projects", "uk-urban-systems-network")
data_dir <- file.path(proj_dir, "data")
fig_dir <- file.path(proj_dir, "figures")
dir.create(data_dir, recursive = TRUE, showWarnings = FALSE)
dir.create(fig_dir, recursive = TRUE, showWarnings = FALSE)

TTWA boundaries (ONS)#

rds_ttwa <- file.path(data_dir, "ttwa_boundaries.rds")
rds_od <- file.path(data_dir, "od_flows_raw.rds")
rds_lookup <- file.path(data_dir, "msoa_ttwa_lookup.rds")

if (!all(file.exists(c(rds_ttwa, rds_od, rds_lookup)))) {
  message("Running live data fetch (ONS boundaries + NOMIS ODWP01EW bulk CSV) …")
  fetch_script <- file.path(proj_dir, "scripts", "fetch_live_data.R")
  if (!file.exists(fetch_script)) {
    stop("Missing fetch script: ", fetch_script)
  }
  status <- system2("Rscript", c(fetch_script), stdout = TRUE, stderr = TRUE)
  if (!is.null(attr(status, "status")) && attr(status, "status") != 0) {
    stop("fetch_live_data.R failed:\n", paste(status, collapse = "\n"))
  }
}

ttwa_boundaries <- readRDS(rds_ttwa)
message("Loaded TTWA boundaries: ", nrow(ttwa_boundaries), " areas")

Loaded TTWA boundaries: 228 areas

Commuting flows (NOMIS)#

# 2021 Census ODWP01EW (bulk download; not available via nomisr API)
od_flows_raw <- readRDS(rds_od)
message("Loaded OD flows: ", nrow(od_flows_raw), " MSOA pairs")

Loaded OD flows: 1741038 MSOA pairs

Preview and summary#

print(head(od_flows_raw))

print(od_flows_raw |> count(source, geo_level))

country_lookup <- tibble(
  country = c("E", "W", "S", "N"),
  country_name = c("England", "Wales", "Scotland", "Northern Ireland")
)

ttwa_country_summary <- ttwa_boundaries |>
  st_drop_geometry() |>
  count(country, name = "n_ttwa") |>
  left_join(country_lookup, by = "country") |>
  select(country_name, n_ttwa)

print(ttwa_country_summary)

  residence_msoa workplace_msoa flow source geo_level
1      E02000001      E02000001  436 EW2021      MSOA
2      E02000001      E02000016    2 EW2021      MSOA
3      E02000001      E02000024    3 EW2021      MSOA
4      E02000001      E02000027    1 EW2021      MSOA
5      E02000001      E02000055    1 EW2021      MSOA
6      E02000001      E02000060    1 EW2021      MSOA

  source geo_level       n
1 EW2021      MSOA 1739598
2 SC2022      TTWA    1440

      country_name n_ttwa
1          England    155
2 Northern Ireland     10
3         Scotland     45
4            Wales     18

Data sources#

Next: Data cleaning →

Contents