Changes in kernel/generic/src/ddi/irq.c [82cbf8c6:9d58539] in mainline

File:

• kernel/generic/src/ddi/irq.c (modified) (6 diffs)

kernel/generic/src/ddi/irq.c

@@ -32 +32 @@
 /**
  * @file
- * @brief IRQ dispatcher
- *
- * This file provides means of connecting IRQs with respective device drivers
- * and logic for dispatching interrupts to IRQ handlers defined by those
- * drivers.
+ * @brief IRQ dispatcher.
+ *
+ * This file provides means of connecting IRQs with particular
+ * devices and logic for dispatching interrupts to IRQ handlers
+ * defined by those devices.
+ *
+ * This code is designed to support:
+ * - multiple devices sharing single IRQ
+ * - multiple IRQs per single device
+ * - multiple instances of the same device
+ *
+ *
+ * Note about architectures.
+ *
+ * Some architectures has the term IRQ well defined. Examples
+ * of such architectures include amd64, ia32 and mips32. Some
+ * other architectures, such as sparc64, don't use the term
+ * at all. In those cases, we boldly step forward and define what
+ * an IRQ is.
+ *
+ * The implementation is generic enough and still allows the
+ * architectures to use the hardware layout effectively.
+ * For instance, on amd64 and ia32, where there is only 16
+ * IRQs, the irq_hash_table can be optimized to a one-dimensional
+ * array. Next, when it is known that the IRQ numbers (aka INR's)
+ * are unique, the claim functions can always return IRQ_ACCEPT.
+ *
+ *
+ * Note about the irq_hash_table.
+ *
+ * The hash table is configured to use two keys: inr and devno.
+ * However, the hash index is computed only from inr. Moreover,
+ * if devno is -1, the match is based on the return value of
+ * the claim() function instead of on devno.
  */
 
 #include <ddi/irq.h>
-#include <adt/hash.h>
 #include <adt/hash_table.h>
 #include <mm/slab.h>
@@ -47 +75 @@
 #include <console/console.h>
 #include <interrupt.h>
-#include <mem.h>
+#include <memstr.h>
 #include <arch.h>
 
-slab_cache_t *irq_slab = NULL;
-
-/** Spinlock protecting the kernel IRQ hash table
+#define KEY_INR    0
+#define KEY_DEVNO  1
+
+/** Spinlock protecting the kernel IRQ hash table.
  *
  * This lock must be taken only when interrupts are disabled.
@@ -62 +91 @@
 static hash_table_t irq_kernel_hash_table;
 
-/** Spinlock protecting the uspace IRQ hash table
+/** Spinlock protecting the uspace IRQ hash table.
  *
  * This lock must be taken only when interrupts are disabled.
@@ -69 +98 @@
 IRQ_SPINLOCK_INITIALIZE(irq_uspace_hash_table_lock);
 
-/** The uspace IRQ hash table */
+/** The uspace IRQ hash table. */
 hash_table_t irq_uspace_hash_table;
 
-static size_t irq_ht_hash(const ht_link_t *);
-static size_t irq_ht_key_hash(void *);
-static bool irq_ht_equal(const ht_link_t *, const ht_link_t *);
-static bool irq_ht_key_equal(void *, const ht_link_t *);
-
-static hash_table_ops_t irq_ht_ops = {
+/**
+ * Hash table operations for cases when we know that
+ * there will be collisions between different keys.
+ *
+ */
+static size_t irq_ht_hash(sysarg_t *key);
+static bool irq_ht_compare(sysarg_t *key, size_t keys, link_t *item);
+static void irq_ht_remove(link_t *item);
+
+static hash_table_operations_t irq_ht_ops = {
         .hash = irq_ht_hash,
-        .key_hash = irq_ht_key_hash,
-        .equal = irq_ht_equal,
-        .key_equal = irq_ht_key_equal
+        .compare = irq_ht_compare,
+        .remove_callback = irq_ht_remove,
 };
 
-/** Last valid INR */
+/**
+ * Hash table operations for cases when we know that
+ * there will be no collisions between different keys.
+ * However, there might be still collisions among
+ * elements with single key (sharing of one IRQ).
+ *
+ */
+static size_t irq_lin_hash(sysarg_t *key);
+static bool irq_lin_compare(sysarg_t *key, size_t keys, link_t *item);
+static void irq_lin_remove(link_t *item);
+
+static hash_table_operations_t irq_lin_ops = {
+        .hash = irq_lin_hash,
+        .compare = irq_lin_compare,
+        .remove_callback = irq_lin_remove,
+};
+
+/** Number of buckets in either of the hash tables. */
+static size_t buckets;
+
+/** Last valid INR. */
 inr_t last_inr = 0;
 
-/** Initialize IRQ subsystem
- *
- * @param inrs    Numbers of unique IRQ numbers or INRs.
- * @param chains  Number of buckets in the hash table.
+/** Initialize IRQ subsystem.
+ *
+ * @param inrs   Numbers of unique IRQ numbers or INRs.
+ * @param chains Number of chains in the hash table.
  *
  */
 void irq_init(size_t inrs, size_t chains)
 {
+        buckets = chains;
         last_inr = inrs - 1;
 
-        irq_slab = slab_cache_create("irq_t", sizeof(irq_t), 0, NULL, NULL,
-            FRAME_ATOMIC);
-        assert(irq_slab);
-
-        hash_table_create(&irq_uspace_hash_table, chains, 0, &irq_ht_ops);
-        hash_table_create(&irq_kernel_hash_table, chains, 0, &irq_ht_ops);
-}
-
-/** Initialize one IRQ structure
- *
- * @param irq  Pointer to the IRQ structure to be initialized.
+        /*
+         * Be smart about the choice of the hash table operations.
+         * In cases in which inrs equals the requested number of
+         * chains (i.e. where there is no collision between
+         * different keys), we can use optimized set of operations.
+         */
+        if (inrs == chains) {
+                hash_table_create(&irq_uspace_hash_table, chains, 2,
+                    &irq_lin_ops);
+                hash_table_create(&irq_kernel_hash_table, chains, 2,
+                    &irq_lin_ops);
+        } else {
+                hash_table_create(&irq_uspace_hash_table, chains, 2,
+                    &irq_ht_ops);
+                hash_table_create(&irq_kernel_hash_table, chains, 2,
+                    &irq_ht_ops);
+        }
+}
+
+/** Initialize one IRQ structure.
+ *
+ * @param irq Pointer to the IRQ structure to be initialized.
  *
  */
@@ -113 +177 @@
 {
         memsetb(irq, sizeof(irq_t), 0);
+        link_initialize(&irq->link);
         irq_spinlock_initialize(&irq->lock, "irq.lock");
+        link_initialize(&irq->notif_cfg.link);
         irq->inr = -1;
+        irq->devno = -1;
         
         irq_initialize_arch(irq);
 }
 
-/** Register IRQ for device
- *
- * The irq structure must be filled with information about the interrupt source
- * and with the claim() function pointer and handler() function pointer.
- *
- * @param irq  IRQ structure belonging to a device.
+/** Register IRQ for device.
+ *
+ * The irq structure must be filled with information
+ * about the interrupt source and with the claim()
+ * function pointer and handler() function pointer.
+ *
+ * @param irq IRQ structure belonging to a device.
+ *
+ * @return True on success, false on failure.
  *
  */
 void irq_register(irq_t *irq)
 {
+        sysarg_t key[] = {
+                (sysarg_t) irq->inr,
+                (sysarg_t) irq->devno
+        };
+        
         irq_spinlock_lock(&irq_kernel_hash_table_lock, true);
         irq_spinlock_lock(&irq->lock, false);
-        hash_table_insert(&irq_kernel_hash_table, &irq->link);
+        hash_table_insert(&irq_kernel_hash_table, key, &irq->link);
         irq_spinlock_unlock(&irq->lock, false);
         irq_spinlock_unlock(&irq_kernel_hash_table_lock, true);
 }
 
-/** Search and lock an IRQ hash table */
-static irq_t *
-irq_dispatch_and_lock_table(hash_table_t *h, irq_spinlock_t *l, inr_t inr)
-{
-        irq_spinlock_lock(l, false);
-        for (ht_link_t *lnk = hash_table_find(h, &inr); lnk;
-            lnk = hash_table_find_next(h, lnk)) {
-                irq_t *irq = hash_table_get_inst(lnk, irq_t, link);
-                irq_spinlock_lock(&irq->lock, false);
-                if (irq->claim(irq) == IRQ_ACCEPT) {
-                        /* leave irq locked */
-                        irq_spinlock_unlock(l, false);
-                        return irq;
-                }
-                irq_spinlock_unlock(&irq->lock, false);
-        }
-        irq_spinlock_unlock(l, false);
+/** Search and lock the uspace IRQ hash table.
+ *
+ */
+static irq_t *irq_dispatch_and_lock_uspace(inr_t inr)
+{
+        link_t *lnk;
+        sysarg_t key[] = {
+                (sysarg_t) inr,
+                (sysarg_t) -1    /* Search will use claim() instead of devno */
+        };
+        
+        irq_spinlock_lock(&irq_uspace_hash_table_lock, false);
+        lnk = hash_table_find(&irq_uspace_hash_table, key);
+        if (lnk) {
+                irq_t *irq = hash_table_get_instance(lnk, irq_t, link);
+                irq_spinlock_unlock(&irq_uspace_hash_table_lock, false);
+                return irq;
+        }
+        irq_spinlock_unlock(&irq_uspace_hash_table_lock, false);
         
         return NULL;
 }
 
-/** Dispatch the IRQ
- *
- * We assume this function is only called from interrupt context (i.e. that
- * interrupts are disabled prior to this call).
- *
- * This function attempts to lookup a fitting IRQ structure. In case of success,
- * return with interrupts disabled and holding the respective structure.
- *
- * @param inr  Interrupt number (aka inr or irq).
- *
- * @return IRQ structure of the respective device
- * @return NULL if no IRQ structure found
+/** Search and lock the kernel IRQ hash table.
+ *
+ */
+static irq_t *irq_dispatch_and_lock_kernel(inr_t inr)
+{
+        link_t *lnk;
+        sysarg_t key[] = {
+                (sysarg_t) inr,
+                (sysarg_t) -1    /* Search will use claim() instead of devno */
+        };
+        
+        irq_spinlock_lock(&irq_kernel_hash_table_lock, false);
+        lnk = hash_table_find(&irq_kernel_hash_table, key);
+        if (lnk) {
+                irq_t *irq = hash_table_get_instance(lnk, irq_t, link);
+                irq_spinlock_unlock(&irq_kernel_hash_table_lock, false);
+                return irq;
+        }
+        irq_spinlock_unlock(&irq_kernel_hash_table_lock, false);
+        
+        return NULL;
+}
+
+/** Dispatch the IRQ.
+ *
+ * We assume this function is only called from interrupt
+ * context (i.e. that interrupts are disabled prior to
+ * this call).
+ *
+ * This function attempts to lookup a fitting IRQ
+ * structure. In case of success, return with interrupts
+ * disabled and holding the respective structure.
+ *
+ * @param inr Interrupt number (aka inr or irq).
+ *
+ * @return IRQ structure of the respective device or NULL.
  *
  */
@@ -174 +275 @@
 {
         /*
-         * If the kernel console override is on, then try first the kernel
-         * handlers and eventually fall back to uspace handlers.
+         * If the kernel console override is on,
+         * then try first the kernel handlers
+         * and eventually fall back to uspace
+         * handlers.
          *
-         * In the usual case the uspace handlers have precedence.
+         * In the usual case the uspace handlers
+         * have precedence.
          */
         
         if (console_override) {
-                irq_t *irq = irq_dispatch_and_lock_table(&irq_kernel_hash_table,
-                    &irq_kernel_hash_table_lock, inr);
+                irq_t *irq = irq_dispatch_and_lock_kernel(inr);
                 if (irq)
                         return irq;
                 
-                return irq_dispatch_and_lock_table(&irq_uspace_hash_table,
-                    &irq_uspace_hash_table_lock, inr);
-        }
-        
-        irq_t *irq = irq_dispatch_and_lock_table(&irq_uspace_hash_table,
-            &irq_uspace_hash_table_lock, inr);
+                return irq_dispatch_and_lock_uspace(inr);
+        }
+        
+        irq_t *irq = irq_dispatch_and_lock_uspace(inr);
         if (irq)
                 return irq;
         
-        return irq_dispatch_and_lock_table(&irq_kernel_hash_table,
-            &irq_kernel_hash_table_lock, inr);
-}
-
-/** Return the hash of the key stored in the item. */
-size_t irq_ht_hash(const ht_link_t *item)
-{
-        irq_t *irq = hash_table_get_inst(item, irq_t, link);
-        return hash_mix(irq->inr);
-}
-
-/** Return the hash of the key. */
-size_t irq_ht_key_hash(void *key)
-{
-        inr_t *inr = (inr_t *) key;
-        return hash_mix(*inr);
-}
-
-/** Return true if the items have the same lookup key. */
-bool irq_ht_equal(const ht_link_t *item1, const ht_link_t *item2)
-{
-        irq_t *irq1 = hash_table_get_inst(item1, irq_t, link);
-        irq_t *irq2 = hash_table_get_inst(item2, irq_t, link);
-        return irq1->inr == irq2->inr;
-}
-
-/** Return true if the key is equal to the item's lookup key. */
-bool irq_ht_key_equal(void *key, const ht_link_t *item)
-{
-        inr_t *inr = (inr_t *) key;
-        irq_t *irq = hash_table_get_inst(item, irq_t, link);
-        return irq->inr == *inr;
+        return irq_dispatch_and_lock_kernel(inr);
+}
+
+/** Compute hash index for the key.
+ *
+ * This function computes hash index into
+ * the IRQ hash table for which there
+ * can be collisions between different
+ * INRs.
+ *
+ * The devno is not used to compute the hash.
+ *
+ * @param key The first of the keys is inr and the second is devno or -1.
+ *
+ * @return Index into the hash table.
+ *
+ */
+size_t irq_ht_hash(sysarg_t key[])
+{
+        inr_t inr = (inr_t) key[KEY_INR];
+        return inr % buckets;
+}
+
+/** Compare hash table element with a key.
+ *
+ * There are two things to note about this function.
+ * First, it is used for the more complex architecture setup
+ * in which there are way too many interrupt numbers (i.e. inr's)
+ * to arrange the hash table so that collisions occur only
+ * among same inrs of different devnos. So the explicit check
+ * for inr match must be done.
+ * Second, if devno is -1, the second key (i.e. devno) is not
+ * used for the match and the result of the claim() function
+ * is used instead.
+ *
+ * This function assumes interrupts are already disabled.
+ *
+ * @param key  Keys (i.e. inr and devno).
+ * @param keys This is 2.
+ * @param item The item to compare the key with.
+ *
+ * @return True on match or false otherwise.
+ *
+ */
+bool irq_ht_compare(sysarg_t key[], size_t keys, link_t *item)
+{
+        irq_t *irq = hash_table_get_instance(item, irq_t, link);
+        inr_t inr = (inr_t) key[KEY_INR];
+        devno_t devno = (devno_t) key[KEY_DEVNO];
+        
+        bool rv;
+        
+        irq_spinlock_lock(&irq->lock, false);
+        if (devno == -1) {
+                /* Invoked by irq_dispatch_and_lock(). */
+                rv = ((irq->inr == inr) &&
+                    (irq->claim(irq) == IRQ_ACCEPT));
+        } else {
+                /* Invoked by irq_find_and_lock(). */
+                rv = ((irq->inr == inr) && (irq->devno == devno));
+        }
+        
+        /* unlock only on non-match */
+        if (!rv)
+                irq_spinlock_unlock(&irq->lock, false);
+        
+        return rv;
+}
+
+/** Unlock IRQ structure after hash_table_remove().
+ *
+ * @param lnk Link in the removed and locked IRQ structure.
+ */
+void irq_ht_remove(link_t *lnk)
+{
+        irq_t *irq __attribute__((unused))
+            = hash_table_get_instance(lnk, irq_t, link);
+        irq_spinlock_unlock(&irq->lock, false);
+}
+
+/** Compute hash index for the key.
+ *
+ * This function computes hash index into
+ * the IRQ hash table for which there
+ * are no collisions between different
+ * INRs.
+ *
+ * @param key The first of the keys is inr and the second is devno or -1.
+ *
+ * @return Index into the hash table.
+ *
+ */
+size_t irq_lin_hash(sysarg_t key[])
+{
+        inr_t inr = (inr_t) key[KEY_INR];
+        return inr;
+}
+
+/** Compare hash table element with a key.
+ *
+ * There are two things to note about this function.
+ * First, it is used for the less complex architecture setup
+ * in which there are not too many interrupt numbers (i.e. inr's)
+ * to arrange the hash table so that collisions occur only
+ * among same inrs of different devnos. So the explicit check
+ * for inr match is not done.
+ * Second, if devno is -1, the second key (i.e. devno) is not
+ * used for the match and the result of the claim() function
+ * is used instead.
+ *
+ * This function assumes interrupts are already disabled.
+ *
+ * @param key  Keys (i.e. inr and devno).
+ * @param keys This is 2.
+ * @param item The item to compare the key with.
+ *
+ * @return True on match or false otherwise.
+ *
+ */
+bool irq_lin_compare(sysarg_t key[], size_t keys, link_t *item)
+{
+        irq_t *irq = list_get_instance(item, irq_t, link);
+        devno_t devno = (devno_t) key[KEY_DEVNO];
+        bool rv;
+        
+        irq_spinlock_lock(&irq->lock, false);
+        if (devno == -1) {
+                /* Invoked by irq_dispatch_and_lock() */
+                rv = (irq->claim(irq) == IRQ_ACCEPT);
+        } else {
+                /* Invoked by irq_find_and_lock() */
+                rv = (irq->devno == devno);
+        }
+        
+        /* unlock only on non-match */
+        if (!rv)
+                irq_spinlock_unlock(&irq->lock, false);
+        
+        return rv;
+}
+
+/** Unlock IRQ structure after hash_table_remove().
+ *
+ * @param lnk Link in the removed and locked IRQ structure.
+ *
+ */
+void irq_lin_remove(link_t *lnk)
+{
+        irq_t *irq __attribute__((unused))
+            = hash_table_get_instance(lnk, irq_t, link);
+        irq_spinlock_unlock(&irq->lock, false);
 }