Changeset 06f81c4 in mainline

Timestamp:

2023-04-16T13:00:39Z (20 months ago)

Author:

Jiří Zárevúcky <zarevucky.jiri@…>

Branches:

master, ticket/834-toolchain-update, topic/msim-upgrade, topic/simplify-dev-export

Children:

d8581611, f3dbe27

Parents:

fbaf6ac

git-author:

Jiří Zárevúcky <zarevucky.jiri@…> (2023-03-28 16:21:30)

git-committer:

Jiří Zárevúcky <zarevucky.jiri@…> (2023-04-16 13:00:39)

Message:

Check cpu_t::fpu_owner directly instead of thread_t::fpu_context_engaged

This results in net reduction in locking.

Location:

kernel/generic

Files:

• include/cpu.h (modified) (1 diff)
• include/proc/thread.h (modified) (1 diff)
• src/cpu/cpu.c (modified) (1 diff)
• src/proc/scheduler.c (modified) (7 diffs)
• src/proc/thread.c (modified) (1 diff)

kernel/generic/include/cpu.h

@@ -97 +97 @@
         cpu_arch_t arch;
 
+#ifdef CONFIG_FPU_LAZY
         IRQ_SPINLOCK_DECLARE(fpu_lock);
         struct thread *fpu_owner;
+#endif
 
         /**

kernel/generic/include/proc/thread.h

@@ -132 +132 @@
         bool fpu_context_exists;
 
-        /*
-         * Defined only if thread doesn't run.
-         * It means that fpu context is in CPU that last time executes this
-         * thread. This disables migration.
-         */
-        bool fpu_context_engaged;
-
         /* The thread will not be migrated if nomigrate is non-zero. */
         unsigned int nomigrate;

kernel/generic/src/cpu/cpu.c

@@ -84 +84 @@
                         cpus[i].id = i;
 
+#ifdef CONFIG_FPU_LAZY
                         irq_spinlock_initialize(&cpus[i].fpu_lock, "cpus[].fpu_lock");
+#endif
                         irq_spinlock_initialize(&cpus[i].tlb_lock, "cpus[].tlb_lock");
 

kernel/generic/src/proc/scheduler.c

@@ -84 +84 @@
 
 #ifdef CONFIG_FPU_LAZY
+        irq_spinlock_lock(&CPU->fpu_lock, true);
+
         if (THREAD == CPU->fpu_owner)
                 fpu_enable();
         else
                 fpu_disable();
+
+        irq_spinlock_unlock(&CPU->fpu_lock, true);
 #elif defined CONFIG_FPU
         fpu_enable();
@@ -133 +137 @@
         /* Save old context */
         if (CPU->fpu_owner != NULL) {
-                irq_spinlock_lock(&CPU->fpu_owner->lock, false);
                 fpu_context_save(&CPU->fpu_owner->fpu_context);
-
-                /* Don't prevent migration */
-                CPU->fpu_owner->fpu_context_engaged = false;
-                irq_spinlock_unlock(&CPU->fpu_owner->lock, false);
                 CPU->fpu_owner = NULL;
         }
 
-        irq_spinlock_lock(&THREAD->lock, false);
         if (THREAD->fpu_context_exists) {
                 fpu_context_restore(&THREAD->fpu_context);
@@ -151 +149 @@
 
         CPU->fpu_owner = THREAD;
-        THREAD->fpu_context_engaged = true;
-        irq_spinlock_unlock(&THREAD->lock, false);
 
         irq_spinlock_unlock(&CPU->fpu_lock, false);
@@ -503 +499 @@
 #ifdef CONFIG_SMP
 
+static inline void fpu_owner_lock(cpu_t *cpu)
+{
+#ifdef CONFIG_FPU_LAZY
+        irq_spinlock_lock(&cpu->fpu_lock, false);
+#endif
+}
+
+static inline void fpu_owner_unlock(cpu_t *cpu)
+{
+#ifdef CONFIG_FPU_LAZY
+        irq_spinlock_unlock(&cpu->fpu_lock, false);
+#endif
+}
+
+static inline thread_t *fpu_owner(cpu_t *cpu)
+{
+#ifdef CONFIG_FPU_LAZY
+        assert(irq_spinlock_locked(&cpu->fpu_lock));
+        return cpu->fpu_owner;
+#else
+        return NULL;
+#endif
+}
+
 static thread_t *steal_thread_from(cpu_t *old_cpu, int i)
 {
@@ -508 +528 @@
         runq_t *new_rq = &CPU->rq[i];
 
-        irq_spinlock_lock(&old_rq->lock, true);
+        ipl_t ipl = interrupts_disable();
+
+        fpu_owner_lock(old_cpu);
+        irq_spinlock_lock(&old_rq->lock, false);
 
         /* Search rq from the back */
@@ -521 +544 @@
                  * FPU context is still in the CPU.
                  */
-                if (thread->stolen || thread->nomigrate || thread->fpu_context_engaged) {
+                if (thread->stolen || thread->nomigrate ||
+                    thread == fpu_owner(old_cpu)) {
                         irq_spinlock_unlock(&thread->lock, false);
                         continue;
                 }
+
+                fpu_owner_unlock(old_cpu);
 
                 thread->stolen = true;
@@ -546 +572 @@
                 old_rq->n--;
                 list_remove(&thread->rq_link);
-
-                irq_spinlock_pass(&old_rq->lock, &new_rq->lock);
+                irq_spinlock_unlock(&old_rq->lock, false);
 
                 /* Append thread to local queue. */
+                irq_spinlock_lock(&new_rq->lock, false);
                 list_append(&thread->rq_link, &new_rq->rq);
                 new_rq->n++;
-
-                irq_spinlock_unlock(&new_rq->lock, true);
+                irq_spinlock_unlock(&new_rq->lock, false);
 
                 atomic_dec(&old_cpu->nrdy);
                 atomic_inc(&CPU->nrdy);
-
+                interrupts_restore(ipl);
                 return thread;
         }
 
-        irq_spinlock_unlock(&old_rq->lock, true);
+        irq_spinlock_unlock(&old_rq->lock, false);
+        fpu_owner_unlock(old_cpu);
+        interrupts_restore(ipl);
         return NULL;
 }

kernel/generic/src/proc/thread.c

@@ -355 +355 @@
 
         thread->fpu_context_exists = false;
-        thread->fpu_context_engaged = false;
 
         odlink_initialize(&thread->lthreads);