feat: add ahfail-pam PAM module with cleanup-based failure detection

Implements a C-ABI PAM shared library that registers a pam_set_data
cleanup callback to detect auth failures and spawn/kill ahfail-display
via a double-fork, without ever touching credentials.

crates/ahfail-pam/Cargo.toml

@@ -2,3 +2,10 @@
 name = "ahfail-pam"
 version = "0.1.0"
 edition = "2021"
+
+[lib]
+name = "ahfail_pam"
+crate-type = ["cdylib", "rlib"]
+
+[dependencies]
+libc = "0.2"

crates/ahfail-pam/build.rs

@@ -0,0 +1,3 @@
+fn main() {
+    println!("cargo:rustc-link-lib=pam");
+}

crates/ahfail-pam/src/lib.rs

@@ -1 +1,180 @@
-pub fn placeholder() {}
+use libc::{c_char, c_int, c_void, pid_t};
+use std::ffi::CString;
+
+// --- PAM constants (platform-specific) ---
+
+pub const PAM_SUCCESS: c_int = 0;
+pub const PAM_IGNORE: c_int = 25;
+
+#[cfg(target_os = "linux")]
+pub const PAM_AUTH_ERR: c_int = 7;
+#[cfg(target_os = "macos")]
+pub const PAM_AUTH_ERR: c_int = 9;
+// Fallback for CI on non-Linux non-macOS (should not occur in production)
+#[cfg(not(any(target_os = "linux", target_os = "macos")))]
+pub const PAM_AUTH_ERR: c_int = 7;
+
+#[cfg(target_os = "linux")]
+pub const PAM_DATA_REPLACE: c_int = 0x20000000u32 as i32;
+#[cfg(target_os = "macos")]
+pub const PAM_DATA_REPLACE: c_int = 0x00000002;
+#[cfg(not(any(target_os = "linux", target_os = "macos")))]
+pub const PAM_DATA_REPLACE: c_int = 0x20000000u32 as i32;
+
+// Default install path for ahfail-display
+#[cfg(target_os = "macos")]
+const DEFAULT_PATH: &str = "/usr/local/lib/ahfail/ahfail-display";
+#[cfg(not(target_os = "macos"))]
+const DEFAULT_PATH: &str = "/usr/lib/ahfail/ahfail-display";
+
+pub fn default_display_path() -> &'static str { DEFAULT_PATH }
+
+pub fn is_failure(s: c_int) -> bool { s & !PAM_DATA_REPLACE == PAM_AUTH_ERR }
+pub fn is_success(s: c_int) -> bool { s & !PAM_DATA_REPLACE == PAM_SUCCESS }
+pub fn is_replace(s: c_int) -> bool { s & PAM_DATA_REPLACE != 0 }
+
+// --- PAM opaque handle ---
+
+#[repr(C)]
+pub struct PamHandle { _private: [u8; 0] }
+
+type CleanupFn = unsafe extern "C" fn(*mut PamHandle, *mut c_void, c_int);
+
+extern "C" {
+    fn pam_set_data(
+        pamh: *mut PamHandle,
+        name: *const c_char,
+        data: *mut c_void,
+        cleanup: Option<CleanupFn>,
+    ) -> c_int;
+}
+
+// --- Cleanup: fires on pam_end() or when data is replaced ---
+unsafe extern "C" fn ahfail_cleanup(
+    _pamh: *mut PamHandle,
+    data: *mut c_void,
+    error_status: c_int,
+) {
+    // Reclaim any stored path string to avoid a leak.
+    let path_override: Option<String> = if data.is_null() {
+        None
+    } else {
+        Some(*Box::from_raw(data as *mut String))
+    };
+
+    if is_replace(error_status) {
+        // Previous attempt failed, same PAM handle; a new attempt is starting.
+        spawn_display(path_override);
+        return;
+    }
+    if is_failure(error_status) {
+        spawn_display(path_override);
+    } else if is_success(error_status) {
+        kill_display();
+    }
+}
+
+// --- Exported PAM module entry points ---
+
+#[no_mangle]
+pub unsafe extern "C" fn pam_sm_authenticate(
+    pamh: *mut PamHandle,
+    _flags: c_int,
+    argc: c_int,
+    argv: *const *const c_char,
+) -> c_int {
+    let args = argc_argv(argc, argv);
+    let display_path = read_display_path_arg(args);
+
+    let key = match CString::new("ahfail") {
+        Ok(k) => k,
+        Err(_) => return PAM_IGNORE,
+    };
+
+    // Store optional path override as heap data for the cleanup function.
+    let path_ptr: *mut c_void = match display_path {
+        Some(p) => Box::into_raw(Box::new(p)) as *mut c_void,
+        None => std::ptr::null_mut(),
+    };
+
+    pam_set_data(pamh, key.as_ptr(), path_ptr, Some(ahfail_cleanup));
+    PAM_IGNORE
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn pam_sm_setcred(
+    _pamh: *mut PamHandle, _flags: c_int, _argc: c_int, _argv: *const *const c_char,
+) -> c_int { PAM_IGNORE }
+
+#[no_mangle]
+pub unsafe extern "C" fn pam_sm_acct_mgmt(
+    _pamh: *mut PamHandle, _flags: c_int, _argc: c_int, _argv: *const *const c_char,
+) -> c_int { PAM_IGNORE }
+
+#[no_mangle]
+pub unsafe extern "C" fn pam_sm_open_session(
+    _pamh: *mut PamHandle, _flags: c_int, _argc: c_int, _argv: *const *const c_char,
+) -> c_int { PAM_IGNORE }
+
+#[no_mangle]
+pub unsafe extern "C" fn pam_sm_close_session(
+    _pamh: *mut PamHandle, _flags: c_int, _argc: c_int, _argv: *const *const c_char,
+) -> c_int { PAM_IGNORE }
+
+#[no_mangle]
+pub unsafe extern "C" fn pam_sm_chauthtok(
+    _pamh: *mut PamHandle, _flags: c_int, _argc: c_int, _argv: *const *const c_char,
+) -> c_int { PAM_IGNORE }
+
+// --- Helpers ---
+
+unsafe fn argc_argv<'a>(argc: c_int, argv: *const *const c_char) -> &'a [*const c_char] {
+    if argv.is_null() || argc <= 0 { return &[]; }
+    std::slice::from_raw_parts(argv, argc as usize)
+}
+
+fn read_display_path_arg(args: &[*const c_char]) -> Option<String> {
+    let prefix = b"display_path=";
+    for &arg in args {
+        if arg.is_null() { continue; }
+        let s = unsafe { std::ffi::CStr::from_ptr(arg) }.to_bytes();
+        if s.starts_with(prefix) {
+            return std::str::from_utf8(&s[prefix.len()..]).ok().map(|s| s.to_string());
+        }
+    }
+    None
+}
+
+fn spawn_display(path_override: Option<String>) {
+    let path = path_override.unwrap_or_else(|| DEFAULT_PATH.to_string());
+    let cpath = match CString::new(path.as_str()) {
+        Ok(p) => p,
+        Err(_) => return,
+    };
+    // Double-fork: grandchild adopted by init, PAM stack not blocked.
+    unsafe {
+        let pid: pid_t = libc::fork();
+        if pid < 0 { return; }
+        if pid > 0 {
+            // Parent: wait for intermediate to exit immediately.
+            libc::waitpid(pid, std::ptr::null_mut(), 0);
+            return;
+        }
+        // Intermediate: fork again then exit immediately.
+        let pid2: pid_t = libc::fork();
+        if pid2 != 0 { libc::_exit(0); }
+        // Grandchild: close inherited fds, exec ahfail-display.
+        libc::close(0); libc::close(1); libc::close(2);
+        let args: [*const c_char; 2] = [cpath.as_ptr(), std::ptr::null()];
+        libc::execv(cpath.as_ptr(), args.as_ptr());
+        libc::_exit(1);
+    }
+}
+
+fn kill_display() {
+    // SIGTERM all ahfail-display processes owned by the current user.
+    let uid = unsafe { libc::getuid() }.to_string();
+    let _ = std::process::Command::new("pkill")
+        .args(["-SIGTERM", "-u", &uid, "ahfail-display"])
+        .spawn();
+}

crates/ahfail-pam/tests/pam_tests.rs

@@ -0,0 +1,21 @@
+use ahfail_pam::{is_failure, is_success, is_replace, PAM_AUTH_ERR, PAM_SUCCESS, PAM_DATA_REPLACE};
+
+#[test]
+fn status_classification() {
+    assert!(is_failure(PAM_AUTH_ERR));
+    assert!(is_success(PAM_SUCCESS));
+    assert!(!is_failure(PAM_SUCCESS));
+    assert!(!is_success(PAM_AUTH_ERR));
+}
+
+#[test]
+fn replace_flag_detected() {
+    let replace_status = PAM_AUTH_ERR | PAM_DATA_REPLACE;
+    assert!(is_replace(replace_status));
+    assert!(!is_replace(PAM_SUCCESS));
+}
+
+#[test]
+fn display_path_default_is_set() {
+    assert!(!ahfail_pam::default_display_path().is_empty());
+}