Changes in kernel/generic/include/atomic.h [174156fd:efed95a3] in mainline

File:

• kernel/generic/include/atomic.h (modified) (2 diffs)

kernel/generic/include/atomic.h

@@ -40 +40 @@
 #include <stdatomic.h>
 
-// TODO: Remove.
-// Before <stdatomic.h> was available, there was only one atomic type
-// equivalent to atomic_size_t. This means that in some places, atomic_t can
-// be replaced with a more appropriate type (e.g. atomic_bool for flags or
-// a signed type for potentially signed values).
-// So atomic_t should be replaced with the most appropriate type on a case by
-// case basis, and after there are no more uses, remove this type.
-typedef atomic_size_t atomic_t;
+/*
+ * Shorthand for relaxed atomic read/write, something that's needed to formally
+ * avoid undefined behavior in cases where we need to read a variable in
+ * different threads and we don't particularly care about ordering
+ * (e.g. statistic printouts). This is most likely translated into the same
+ * assembly instructions as regular read/writes.
+ */
+#define atomic_set_unordered(var, val) atomic_store_explicit((var), (val), memory_order_relaxed)
+#define atomic_get_unordered(var) atomic_load_explicit((var), memory_order_relaxed)
 
 #define atomic_predec(val) \
@@ -72 +73 @@
             (new_val), memory_order_relaxed)
 
+#if __64_BITS__
+
+typedef struct {
+        atomic_uint_fast64_t value;
+} atomic_time_stat_t;
+
+#define ATOMIC_TIME_INITIALIZER() (atomic_time_stat_t) {}
+
+static inline void atomic_time_increment(atomic_time_stat_t *time, int a)
+{
+        /*
+         * We require increments to be synchronized with each other, so we
+         * can use ordinary reads and writes instead of a more expensive atomic
+         * read-modify-write operations.
+         */
+        uint64_t v = atomic_load_explicit(&time->value, memory_order_relaxed);
+        atomic_store_explicit(&time->value, v + a, memory_order_relaxed);
+}
+
+static inline uint64_t atomic_time_read(atomic_time_stat_t *time)
+{
+        return atomic_load_explicit(&time->value, memory_order_relaxed);
+}
+
+#else
+
+/**
+ * A monotonically increasing 64b time statistic.
+ * Increments must be synchronized with each other (or limited to a single
+ * thread/CPU), but reads can be performed from any thread.
+ *
+ */
+typedef struct {
+        uint64_t true_value;
+        atomic_uint_fast32_t high1;
+        atomic_uint_fast32_t high2;
+        atomic_uint_fast32_t low;
+} atomic_time_stat_t;
+
+#define ATOMIC_TIME_INITIALIZER() (atomic_time_stat_t) {}
+
+static inline void atomic_time_increment(atomic_time_stat_t *time, int a)
+{
+        /*
+         * On 32b architectures, we can't rely on 64b memory reads/writes being
+         * architecturally atomic, but we also don't want to pay the cost of
+         * emulating atomic reads/writes, so instead we split value in half
+         * and perform some ordering magic to make sure readers always get
+         * consistent value.
+         */
+
+        /* true_value is only used by the writer, so this need not be atomic. */
+        uint64_t val = time->true_value;
+        uint32_t old_high = val >> 32;
+        val += a;
+        uint32_t new_high = val >> 32;
+        time->true_value = val;
+
+        /* Tell GCC that the first branch is far more likely than the second. */
+        if (__builtin_expect(old_high == new_high, 1)) {
+                /* If the high half didn't change, we need not bother with barriers. */
+                atomic_store_explicit(&time->low, (uint32_t) val, memory_order_relaxed);
+        } else {
+                /*
+                 * If both halves changed, extra ordering is necessary.
+                 * The idea is that if reader reads high1 and high2 with the same value,
+                 * it is guaranteed that they read the correct low half for that value.
+                 *
+                 * This is the same sequence that is used by userspace to read clock.
+                 */
+                atomic_store_explicit(&time->high1, new_high, memory_order_relaxed);
+                atomic_store_explicit(&time->low, (uint32_t) val, memory_order_release);
+                atomic_store_explicit(&time->high2, new_high, memory_order_release);
+        }
+}
+
+static inline uint64_t atomic_time_read(atomic_time_stat_t *time)
+{
+        uint32_t high2 = atomic_load_explicit(&time->high2, memory_order_acquire);
+        uint32_t low = atomic_load_explicit(&time->low, memory_order_acquire);
+        uint32_t high1 = atomic_load_explicit(&time->high1, memory_order_relaxed);
+
+        if (high1 != high2)
+                low = 0;
+
+        /* If the values differ, high1 is always the newer value. */
+        return (uint64_t) high1 << 32 | (uint64_t) low;
+}
+
+#endif /* __64_BITS__ */
+
 #endif