diff --git a/fontique/src/backend/fontconfig/config.rs b/fontique/src/backend/fontconfig/config.rs
index d16532041..a1117547a 100644
--- a/fontique/src/backend/fontconfig/config.rs
+++ b/fontique/src/backend/fontconfig/config.rs
@@ -30,6 +30,7 @@ pub fn parse_config(path: &Path, sink: &mut impl ParserSink) {
     if root.tag_name().name() != "fontconfig" {
         return;
     }
+    let mut has_cachedir = false;
     let mut prefer = vec![];
     'outer: for child in root.children() {
         match child.tag_name().name() {
@@ -65,6 +66,9 @@ pub fn parse_config(path: &Path, sink: &mut impl ParserSink) {
             "cachedir" => {
                 if let Some(path) = resolve_dir(child, path) {
                     sink.cache_path(&path);
+                    if !has_cachedir {
+                        has_cachedir = true;
+                    }
                 }
             }
             "include" => {
@@ -121,6 +125,10 @@ pub fn parse_config(path: &Path, sink: &mut impl ParserSink) {
             _ => {}
         }
     }
+
+    if !has_cachedir {
+        sink.cache_path(&fc_xdg_dir_home("XDG_CACHE_HOME"));
+    }
 }
 
 /// Families we care about for aliases.
@@ -177,16 +185,13 @@ fn include_config(path: &Path, sink: &mut impl ParserSink) -> std::io::Result<()
 
 fn resolve_dir(node: Node<'_, '_>, config_file_path: impl AsRef<Path>) -> Option<PathBuf> {
     let dir_path = node.text()?;
-    let (xdg_env, xdg_fallback) = match node.tag_name().name() {
-        "include" => ("XDG_CONFIG_HOME", "~/.config"),
-        "cachedir" => ("XDG_CACHE_HOME", "~/.cache"),
+    let xdg_env = match node.tag_name().name() {
+        "include" => "XDG_CONFIG_HOME",
+        "cachedir" => "XDG_CACHE_HOME",
         _ => return None,
     };
     let path = match node.attribute("prefix") {
-        Some("xdg") => {
-            PathBuf::from(std::env::var(xdg_env).unwrap_or_else(|_| xdg_fallback.into()))
-                .join(dir_path)
-        }
+        Some("xdg") => xdg_dir_home(xdg_env).join(dir_path),
         _ => {
             if dir_path.starts_with('/') {
                 dir_path.into()
@@ -198,12 +203,7 @@ fn resolve_dir(node: Node<'_, '_>, config_file_path: impl AsRef<Path>) -> Option
             }
         }
     };
-    Some(if let Ok(stripped_path) = path.strip_prefix("~") {
-        let home = config_home().unwrap_or("/".to_string());
-        Path::new(&home).join(stripped_path)
-    } else {
-        path
-    })
+    Some(expand_home(path))
 }
 
 /// Get the location to user home directory.
@@ -219,3 +219,45 @@ fn config_home() -> Result<String, std::env::VarError> {
     }
     home
 }
+
+/// Expand initial "~" in path expands to your home directory.
+fn expand_home(path: PathBuf) -> PathBuf {
+    match (path.strip_prefix("~"), config_home()) {
+        (Ok(stripped_path), Ok(home)) => Path::new(&home).join(stripped_path),
+        (Ok(_), Err(e)) => {
+            panic!("Failed to get the location to user home directory: {:?}", e)
+        }
+        (_, _) => path,
+    }
+}
+
+fn xdg_dir_home(environ: &str) -> PathBuf {
+    fn default_path(environ: &str) -> PathBuf {
+        let fallback = match environ {
+            "XDG_CONFIG_HOME" => "~/.config",
+            "XDG_CACHE_HOME" => "~/.cache",
+            _ => unreachable!(),
+        };
+        PathBuf::from(fallback)
+    }
+
+    fn abspath(path: String) -> Option<PathBuf> {
+        let path = PathBuf::from(path);
+        if path.is_absolute() || path.strip_prefix("~").is_ok() {
+            Some(path)
+        } else {
+            None
+        }
+    }
+
+    let dir_home = std::env::var(environ)
+        .ok()
+        .and_then(abspath)
+        .unwrap_or(default_path(environ));
+
+    expand_home(dir_home)
+}
+
+pub(crate) fn fc_xdg_dir_home(environ: &str) -> PathBuf {
+    xdg_dir_home(environ).join("fontconfig")
+}
diff --git a/fontique/src/backend/fontconfig/mod.rs b/fontique/src/backend/fontconfig/mod.rs
index 4fd3d0ecc..a7a4426a6 100644
--- a/fontique/src/backend/fontconfig/mod.rs
+++ b/fontique/src/backend/fontconfig/mod.rs
@@ -90,23 +90,12 @@ impl SystemFonts {
         // First, parse the raw config files
         let mut config = Config::default();
         config::parse_config("/etc/fonts/fonts.conf".as_ref(), &mut config);
-        if let Ok(xdg_config_home) = std::env::var("XDG_CONFIG_HOME") {
-            config::parse_config(
-                PathBuf::from(xdg_config_home)
-                    .as_path()
-                    .join("fontconfig/fonts.conf")
-                    .as_path(),
-                &mut config,
-            );
-        } else if let Ok(user_home) = std::env::var("HOME") {
-            config::parse_config(
-                PathBuf::from(user_home)
-                    .as_path()
-                    .join(".config/fontconfig/fonts.conf")
-                    .as_path(),
-                &mut config,
-            );
-        }
+        config::parse_config(
+            config::fc_xdg_dir_home("XDG_CONFIG_HOME")
+                .join("fonts.conf")
+                .as_path(),
+            &mut config,
+        );
 
         // Extract all font/family metadata from the cache files
         cache::parse_caches(&config.cache_dirs, |font| {