Changes in / [ed7e057:a5b5f17] in mainline


Ignore:
Location:
kernel
Files:
1 deleted
16 edited

Legend:

Unmodified
Added
Removed

  • kernel/arch/sparc64/src/proc/sun4u/scheduler.c

    red7e057 ra5b5f17  
    7676{
    7777        if (THREAD->uspace) {
     78                asm volatile ("flushw");
     79
    7880                /* sample the state of the userspace window buffer */
    7981                THREAD->arch.uspace_window_buffer =

  • kernel/arch/sparc64/src/proc/sun4v/scheduler.c

    red7e057 ra5b5f17  
    6868{
    6969        if (THREAD->uspace) {
     70                asm volatile ("flushw");
     71
    7072                /* sample the state of the userspace window buffer */
    7173                THREAD->arch.uspace_window_buffer =

  • kernel/generic/include/atomic.h

    red7e057 ra5b5f17  
    3939#include <typedefs.h>
    4040#include <stdatomic.h>
     41
     42/*
     43 * Shorthand for relaxed atomic read/write, something that's needed to formally
     44 * avoid undefined behavior in cases where we need to read a variable in
     45 * different threads and we don't particularly care about ordering
     46 * (e.g. statistic printouts). This is most likely translated into the same
     47 * assembly instructions as regular read/writes.
     48 */
     49#define atomic_set_unordered(var, val) atomic_store_explicit((var), (val), memory_order_relaxed)
     50#define atomic_get_unordered(var) atomic_load_explicit((var), memory_order_relaxed)
    4151
    4252#define atomic_predec(val) \

  • kernel/generic/include/cpu.h

    red7e057 ra5b5f17  
    7474        bool idle;
    7575        uint64_t last_cycle;
     76
     77        context_t scheduler_context;
     78
     79        struct thread *prev_thread;
    7680} cpu_local_t;
    7781

  • kernel/generic/include/proc/scheduler.h

    red7e057 ra5b5f17  
    6464extern void scheduler_enter(state_t);
    6565
     66extern void thread_main_func(void);
     67
    6668/*
    6769 * To be defined by architectures.

  • kernel/generic/include/proc/thread.h

    red7e057 ra5b5f17  
    9595        waitq_t join_wq;
    9696
    97         /** Lock protecting thread structure.
     97        /** Thread accounting. */
     98        atomic_time_stat_t ucycles;
     99        atomic_time_stat_t kcycles;
     100
     101        /** Architecture-specific data. */
     102        thread_arch_t arch;
     103
     104#ifdef CONFIG_UDEBUG
     105        /**
     106         * If true, the scheduler will print a stack trace
     107         * to the kernel console upon scheduling this thread.
     108         */
     109        atomic_int_fast8_t btrace;
     110
     111        /** Debugging stuff */
     112        udebug_thread_t udebug;
     113#endif /* CONFIG_UDEBUG */
     114
     115        /*
     116         * Immutable fields.
    98117         *
    99          * Protects the whole thread structure except fields listed above.
    100          */
    101         IRQ_SPINLOCK_DECLARE(lock);
    102 
    103         char name[THREAD_NAME_BUFLEN];
     118         * These fields are only modified during initialization, and are not
     119         * changed at any time between initialization and destruction.
     120         * Can be accessed without synchronization in most places.
     121         */
     122
     123        /** Thread ID. */
     124        thread_id_t tid;
    104125
    105126        /** Function implementing the thread. */
    … …  
    108129        void *thread_arg;
    109130
     131        char name[THREAD_NAME_BUFLEN];
     132
     133        /** Thread is executed in user space. */
     134        bool uspace;
     135
     136        /** Thread doesn't affect accumulated accounting. */
     137        bool uncounted;
     138
     139        /** Containing task. */
     140        task_t *task;
     141
     142        /** Thread's kernel stack. */
     143        uint8_t *kstack;
     144
     145        /*
     146         * Local fields.
     147         *
     148         * These fields can be safely accessed from code that _controls execution_
     149         * of this thread. Code controls execution of a thread if either:
     150         *  - it runs in the context of said thread AND interrupts are disabled
     151         *    (interrupts can and will access these fields)
     152         *  - the thread is not running, and the code accessing it can legally
     153         *    add/remove the thread to/from a runqueue, i.e., either:
     154         *    - it is allowed to enqueue thread in a new runqueue
     155         *    - it holds the lock to the runqueue containing the thread
     156         *
     157         */
     158
    110159        /**
    111160         * From here, the stored context is restored
    … …  
    114163        context_t saved_context;
    115164
     165        // TODO: we only need one of the two bools below
     166
    116167        /**
    117168         * True if this thread is executing copy_from_uspace().
    … …  
    126177        bool in_copy_to_uspace;
    127178
     179        /*
     180         * FPU context is a special case. If lazy FPU switching is disabled,
     181         * it acts as a regular local field. However, if lazy switching is enabled,
     182         * the context is synchronized via CPU->fpu_lock
     183         */
    128184#ifdef CONFIG_FPU
    129185        fpu_context_t fpu_context;
    … …  
    134190        unsigned int nomigrate;
    135191
    136         /** Thread state. */
    137         state_t state;
    138 
    139         /** Thread CPU. */
    140         cpu_t *cpu;
    141         /** Containing task. */
    142         task_t *task;
    143192        /** Thread was migrated to another CPU and has not run yet. */
    144193        bool stolen;
    145         /** Thread is executed in user space. */
    146         bool uspace;
    147 
    148         /** Thread accounting. */
    149         uint64_t ucycles;
    150         uint64_t kcycles;
     194
     195        /**
     196         * Thread state (state_t).
     197         * This is atomic because we read it via some commands for debug output,
     198         * otherwise it could just be a regular local.
     199         */
     200        atomic_int_fast32_t state;
     201
     202        /** Thread CPU. */
     203        _Atomic(cpu_t *) cpu;
     204
     205        /** Thread's priority. Implemented as index to CPU->rq */
     206        atomic_int_fast32_t priority;
     207
    151208        /** Last sampled cycle. */
    152209        uint64_t last_cycle;
    153         /** Thread doesn't affect accumulated accounting. */
    154         bool uncounted;
    155 
    156         /** Thread's priority. Implemented as index to CPU->rq */
    157         int priority;
    158         /** Thread ID. */
    159         thread_id_t tid;
    160 
    161         /** Architecture-specific data. */
    162         thread_arch_t arch;
    163 
    164         /** Thread's kernel stack. */
    165         uint8_t *kstack;
    166 
    167 #ifdef CONFIG_UDEBUG
    168         /**
    169          * If true, the scheduler will print a stack trace
    170          * to the kernel console upon scheduling this thread.
    171          */
    172         bool btrace;
    173 
    174         /** Debugging stuff */
    175         udebug_thread_t udebug;
    176 #endif /* CONFIG_UDEBUG */
    177210} thread_t;
    178211
    … …  
    186219extern void thread_attach(thread_t *, task_t *);
    187220extern void thread_start(thread_t *);
    188 extern void thread_ready(thread_t *);
     221extern void thread_requeue_sleeping(thread_t *);
    189222extern void thread_exit(void) __attribute__((noreturn));
    190223extern void thread_interrupt(thread_t *);

  • kernel/generic/meson.build

    red7e057 ra5b5f17  
    9595        'src/mm/malloc.c',
    9696        'src/mm/reserve.c',
    97         'src/preempt/preemption.c',
    9897        'src/printf/printf.c',
    9998        'src/printf/snprintf.c',

  • kernel/generic/src/interrupt/interrupt.c

    red7e057 ra5b5f17  
    114114
    115115        /* Account user cycles */
    116         if (THREAD) {
    117                 irq_spinlock_lock(&THREAD->lock, false);
     116        if (THREAD)
    118117                thread_update_accounting(true);
    119                 irq_spinlock_unlock(&THREAD->lock, false);
    120         }
    121118
    122119        /* Account CPU usage if it woke up from sleep */
    … …  
    155152
    156153        /* Do not charge THREAD for exception cycles */
    157         if (THREAD) {
    158                 irq_spinlock_lock(&THREAD->lock, false);
     154        if (THREAD)
    159155                THREAD->last_cycle = end_cycle;
    160                 irq_spinlock_unlock(&THREAD->lock, false);
    161         }
    162156#else
    163157        panic("No space for any exception handler, yet we want to handle some exception.");

  • kernel/generic/src/main/main.c

    red7e057 ra5b5f17  
    287287         * starting the thread of kernel threads.
    288288         */
    289         scheduler_run();
     289        current_copy(CURRENT, (current_t *) CPU_LOCAL->stack);
     290        context_replace(scheduler_run, CPU_LOCAL->stack, STACK_SIZE);
    290291        /* not reached */
    291292}
    … …  
    327328        ARCH_OP(post_cpu_init);
    328329
    329         current_copy(CURRENT, (current_t *) CPU_LOCAL->stack);
    330 
    331330        /*
    332331         * If we woke kmp up before we left the kernel stack, we could
    … …  
    334333         * switch to this cpu's private stack prior to waking kmp up.
    335334         */
     335        current_copy(CURRENT, (current_t *) CPU_LOCAL->stack);
    336336        context_replace(main_ap_separated_stack, CPU_LOCAL->stack, STACK_SIZE);
    337337        /* not reached */

  • kernel/generic/src/proc/scheduler.c

    red7e057 ra5b5f17  
    11/*
    22 * Copyright (c) 2010 Jakub Jermar
     3 * Copyright (c) 2023 Jiří Zárevúcky
    34 * All rights reserved.
    45 *
    … …  
    5051#include <time/delay.h>
    5152#include <arch/asm.h>
    52 #include <arch/faddr.h>
    5353#include <arch/cycle.h>
    5454#include <atomic.h>
    … …  
    6666#include <stacktrace.h>
    6767
    68 static void scheduler_separated_stack(void);
    69 
    7068atomic_size_t nrdy;  /**< Number of ready threads in the system. */
    7169
    … …  
    227225static void relink_rq(int start)
    228226{
     227        assert(interrupts_disabled());
     228
    229229        if (CPU_LOCAL->current_clock_tick < CPU_LOCAL->relink_deadline)
    230230                return;
    … …  
    302302}
    303303
    304 void scheduler_run(void)
    305 {
    306         assert(interrupts_disabled());
    307         assert(THREAD == NULL);
    308         assert(CPU != NULL);
    309 
    310         current_copy(CURRENT, (current_t *) CPU_LOCAL->stack);
    311         context_replace(scheduler_separated_stack, CPU_LOCAL->stack, STACK_SIZE);
    312         unreachable();
    313 }
    314 
    315304/** Things to do before we switch to THREAD context.
    316305 */
    … …  
    321310        switch_task(THREAD->task);
    322311
    323         irq_spinlock_lock(&THREAD->lock, false);
    324         THREAD->state = Running;
    325         THREAD->cpu = CPU;
    326         THREAD->priority = rq_index;  /* Correct rq index */
     312        assert(atomic_get_unordered(&THREAD->cpu) == CPU);
     313
     314        atomic_set_unordered(&THREAD->state, Running);
     315        atomic_set_unordered(&THREAD->priority, rq_index);  /* Correct rq index */
    327316
    328317        /*
    … …  
    335324        log(LF_OTHER, LVL_DEBUG,
    336325            "cpu%u: tid %" PRIu64 " (priority=%d, ticks=%" PRIu64
    337             ", nrdy=%zu)", CPU->id, THREAD->tid, THREAD->priority,
     326            ", nrdy=%zu)", CPU->id, THREAD->tid, rq_index,
    338327            THREAD->ticks, atomic_load(&CPU->nrdy));
    339328#endif
    … …  
    350339
    351340#ifdef CONFIG_UDEBUG
    352         if (THREAD->btrace) {
     341        if (atomic_get_unordered(&THREAD->btrace)) {
    353342                istate_t *istate = THREAD->udebug.uspace_state;
    354343                if (istate != NULL) {
    355344                        printf("Thread %" PRIu64 " stack trace:\n", THREAD->tid);
    356345                        stack_trace_istate(istate);
     346                } else {
     347                        printf("Thread %" PRIu64 " interrupt state not available\n", THREAD->tid);
    357348                }
    358349
    359                 THREAD->btrace = false;
     350                atomic_set_unordered(&THREAD->btrace, false);
    360351        }
    361352#endif
    … …  
    374365}
    375366
    376 static void cleanup_after_thread(thread_t *thread, state_t out_state)
     367static void add_to_rq(thread_t *thread, cpu_t *cpu, int i)
     368{
     369        /* Add to the appropriate runqueue. */
     370        runq_t *rq = &cpu->rq[i];
     371
     372        irq_spinlock_lock(&rq->lock, false);
     373        list_append(&thread->rq_link, &rq->rq);
     374        rq->n++;
     375        irq_spinlock_unlock(&rq->lock, false);
     376
     377        atomic_inc(&nrdy);
     378        atomic_inc(&cpu->nrdy);
     379}
     380
     381/** Requeue a thread that was just preempted on this CPU.
     382 */
     383static void thread_requeue_preempted(thread_t *thread)
     384{
     385        assert(interrupts_disabled());
     386        assert(atomic_get_unordered(&thread->state) == Running);
     387        assert(atomic_get_unordered(&thread->cpu) == CPU);
     388
     389        int prio = atomic_get_unordered(&thread->priority);
     390
     391        if (prio < RQ_COUNT - 1) {
     392                prio++;
     393                atomic_set_unordered(&thread->priority, prio);
     394        }
     395
     396        atomic_set_unordered(&thread->state, Ready);
     397
     398        add_to_rq(thread, CPU, prio);
     399}
     400
     401void thread_requeue_sleeping(thread_t *thread)
     402{
     403        ipl_t ipl = interrupts_disable();
     404
     405        assert(atomic_get_unordered(&thread->state) == Sleeping || atomic_get_unordered(&thread->state) == Entering);
     406
     407        atomic_set_unordered(&thread->priority, 0);
     408        atomic_set_unordered(&thread->state, Ready);
     409
     410        /* Prefer the CPU on which the thread ran last */
     411        cpu_t *cpu = atomic_get_unordered(&thread->cpu);
     412
     413        if (!cpu) {
     414                cpu = CPU;
     415                atomic_set_unordered(&thread->cpu, CPU);
     416        }
     417
     418        add_to_rq(thread, cpu, 0);
     419
     420        interrupts_restore(ipl);
     421}
     422
     423static void cleanup_after_thread(thread_t *thread)
    377424{
    378425        assert(CURRENT->mutex_locks == 0);
    … …  
    381428        int expected;
    382429
    383         switch (out_state) {
     430        switch (atomic_get_unordered(&thread->state)) {
    384431        case Running:
    385                 thread_ready(thread);
     432                thread_requeue_preempted(thread);
    386433                break;
    387434
    … …  
    407454                        assert(expected == SLEEP_WOKE);
    408455                        /* The thread has already been woken up, requeue immediately. */
    409                         thread_ready(thread);
     456                        thread_requeue_sleeping(thread);
    410457                }
    411458                break;
    … …  
    416463                 */
    417464                panic("tid%" PRIu64 ": unexpected state %s.",
    418                     thread->tid, thread_states[thread->state]);
     465                    thread->tid, thread_states[atomic_get_unordered(&thread->state)]);
    419466                break;
    420467        }
    421468}
    422469
    423 /** The scheduler
    424  *
    425  * The thread scheduling procedure.
    426  * Passes control directly to
    427  * scheduler_separated_stack().
    428  *
    429  */
     470/** Switch to scheduler context to let other threads run. */
    430471void scheduler_enter(state_t new_state)
    431472{
    … …  
    435476        assert(THREAD != NULL);
    436477
    437         fpu_cleanup();
    438 
    439         irq_spinlock_lock(&THREAD->lock, false);
    440         THREAD->state = new_state;
    441 
    442         /* Update thread kernel accounting */
    443         THREAD->kcycles += get_cycle() - THREAD->last_cycle;
    444 
    445         if (new_state == Sleeping) {
    446                 /* Prefer the thread after it's woken up. */
    447                 THREAD->priority = -1;
    448         }
    449 
    450         /*
    451          * Through the 'CURRENT' structure, we keep track of THREAD, TASK, CPU, AS
    452          * and preemption counter. At this point CURRENT could be coming either
    453          * from THREAD's or CPU's stack.
    454          *
    455          */
    456         current_copy(CURRENT, (current_t *) CPU_LOCAL->stack);
    457 
    458         /*
    459          * We may not keep the old stack.
    460          * Reason: If we kept the old stack and got blocked, for instance, in
    461          * find_best_thread(), the old thread could get rescheduled by another
    462          * CPU and overwrite the part of its own stack that was also used by
    463          * the scheduler on this CPU.
    464          *
    465          * Moreover, we have to bypass the compiler-generated POP sequence
    466          * which is fooled by SP being set to the very top of the stack.
    467          * Therefore the scheduler() function continues in
    468          * scheduler_separated_stack().
    469          *
    470          */
    471         context_t ctx;
    472         context_create(&ctx, scheduler_separated_stack,
    473             CPU_LOCAL->stack, STACK_SIZE);
    474 
    475         /* Switch to scheduler context and store current thread's context. */
    476         context_swap(&THREAD->saved_context, &ctx);
    477 
    478         /* Returned from scheduler. */
    479 
    480         irq_spinlock_unlock(&THREAD->lock, false);
    481         interrupts_restore(ipl);
    482 }
    483 
    484 /** Scheduler stack switch wrapper
    485  *
    486  * Second part of the scheduler() function
    487  * using new stack. Handling the actual context
    488  * switch to a new thread.
    489  *
    490  */
    491 void scheduler_separated_stack(void)
    492 {
    493         assert((!THREAD) || (irq_spinlock_locked(&THREAD->lock)));
    494         assert(CPU != NULL);
    495         assert(interrupts_disabled());
    496 
    497478        if (atomic_load(&haltstate))
    498479                halt();
    499480
    500         if (THREAD) {
    501                 /*
    502                  * On Sparc, this saves some extra userspace state that's not
    503                  * covered by context_save()/context_restore().
    504                  */
    505                 after_thread_ran_arch();
    506 
    507                 state_t state = THREAD->state;
    508                 irq_spinlock_unlock(&THREAD->lock, false);
    509 
    510                 cleanup_after_thread(THREAD, state);
    511 
     481        /* Check if we have a thread to switch to. */
     482
     483        int rq_index;
     484        thread_t *new_thread = try_find_thread(&rq_index);
     485
     486        if (new_thread == NULL && new_state == Running) {
     487                /* No other thread to run, but we still have work to do here. */
     488                interrupts_restore(ipl);
     489                return;
     490        }
     491
     492        atomic_set_unordered(&THREAD->state, new_state);
     493
     494        /* Update thread kernel accounting */
     495        atomic_time_increment(&THREAD->kcycles, get_cycle() - THREAD->last_cycle);
     496
     497        fpu_cleanup();
     498
     499        /*
     500         * On Sparc, this saves some extra userspace state that's not
     501         * covered by context_save()/context_restore().
     502         */
     503        after_thread_ran_arch();
     504
     505        if (new_thread) {
     506                thread_t *old_thread = THREAD;
     507                CPU_LOCAL->prev_thread = old_thread;
     508                THREAD = new_thread;
     509                /* No waiting necessary, we can switch to the new thread directly. */
     510                prepare_to_run_thread(rq_index);
     511
     512                current_copy(CURRENT, (current_t *) new_thread->kstack);
     513                context_swap(&old_thread->saved_context, &new_thread->saved_context);
     514        } else {
     515                /*
     516                 * A new thread isn't immediately available, switch to a separate
     517                 * stack to sleep or do other idle stuff.
     518                 */
     519                current_copy(CURRENT, (current_t *) CPU_LOCAL->stack);
     520                context_swap(&THREAD->saved_context, &CPU_LOCAL->scheduler_context);
     521        }
     522
     523        assert(CURRENT->mutex_locks == 0);
     524        assert(interrupts_disabled());
     525
     526        /* Check if we need to clean up after another thread. */
     527        if (CPU_LOCAL->prev_thread) {
     528                cleanup_after_thread(CPU_LOCAL->prev_thread);
     529                CPU_LOCAL->prev_thread = NULL;
     530        }
     531
     532        interrupts_restore(ipl);
     533}
     534
     535/** Enter main scheduler loop. Never returns.
     536 *
     537 * This function switches to a runnable thread as soon as one is available,
     538 * after which it is only switched back to if a thread is stopping and there is
     539 * no other thread to run in its place. We need a separate context for that
     540 * because we're going to block the CPU, which means we need another context
     541 * to clean up after the previous thread.
     542 */
     543void scheduler_run(void)
     544{
     545        assert(interrupts_disabled());
     546
     547        assert(CPU != NULL);
     548        assert(TASK == NULL);
     549        assert(THREAD == NULL);
     550        assert(interrupts_disabled());
     551
     552        while (!atomic_load(&haltstate)) {
     553                assert(CURRENT->mutex_locks == 0);
     554
     555                int rq_index;
     556                THREAD = find_best_thread(&rq_index);
     557                prepare_to_run_thread(rq_index);
     558
     559                /*
     560                 * Copy the knowledge of CPU, TASK, THREAD and preemption counter to
     561                 * thread's stack.
     562                 */
     563                current_copy(CURRENT, (current_t *) THREAD->kstack);
     564
     565                /* Switch to thread context. */
     566                context_swap(&CPU_LOCAL->scheduler_context, &THREAD->saved_context);
     567
     568                /* Back from another thread. */
     569                assert(CPU != NULL);
     570                assert(THREAD != NULL);
     571                assert(CURRENT->mutex_locks == 0);
     572                assert(interrupts_disabled());
     573
     574                cleanup_after_thread(THREAD);
     575
     576                /*
     577                 * Necessary because we're allowing interrupts in find_best_thread(),
     578                 * so we need to avoid other code referencing the thread we left.
     579                 */
    512580                THREAD = NULL;
    513581        }
    514582
    515         int rq_index;
    516         THREAD = find_best_thread(&rq_index);
    517 
    518         prepare_to_run_thread(rq_index);
    519 
    520         /*
    521          * Copy the knowledge of CPU, TASK, THREAD and preemption counter to
    522          * thread's stack.
    523          */
    524         current_copy(CURRENT, (current_t *) THREAD->kstack);
    525 
    526         context_restore(&THREAD->saved_context);
     583        halt();
     584}
     585
     586/** Thread wrapper.
     587 *
     588 * This wrapper is provided to ensure that a starting thread properly handles
     589 * everything it needs to do when first scheduled, and when it exits.
     590 */
     591void thread_main_func(void)
     592{
     593        assert(interrupts_disabled());
     594
     595        void (*f)(void *) = THREAD->thread_code;
     596        void *arg = THREAD->thread_arg;
     597
     598        /* This is where each thread wakes up after its creation */
     599
     600        /* Check if we need to clean up after another thread. */
     601        if (CPU_LOCAL->prev_thread) {
     602                cleanup_after_thread(CPU_LOCAL->prev_thread);
     603                CPU_LOCAL->prev_thread = NULL;
     604        }
     605
     606        interrupts_enable();
     607
     608        f(arg);
     609
     610        thread_exit();
    527611
    528612        /* Not reached */
    … …  
    550634        list_foreach_rev(old_rq->rq, rq_link, thread_t, thread) {
    551635
    552                 irq_spinlock_lock(&thread->lock, false);
    553 
    554636                /*
    555637                 * Do not steal CPU-wired threads, threads
    … …  
    558640                 * FPU context is still in the CPU.
    559641                 */
    560                 if (thread->stolen || thread->nomigrate ||
    561                     thread == fpu_owner) {
    562                         irq_spinlock_unlock(&thread->lock, false);
     642                if (thread->stolen || thread->nomigrate || thread == fpu_owner) {
    563643                        continue;
    564644                }
    565645
    566646                thread->stolen = true;
    567                 thread->cpu = CPU;
    568 
    569                 irq_spinlock_unlock(&thread->lock, false);
     647                atomic_set_unordered(&thread->cpu, CPU);
    570648
    571649                /*
    … …  
    712790                            thread) {
    713791                                printf("%" PRIu64 "(%s) ", thread->tid,
    714                                     thread_states[thread->state]);
     792                                    thread_states[atomic_get_unordered(&thread->state)]);
    715793                        }
    716794                        printf("\n");

  • kernel/generic/src/proc/task.c

    red7e057 ra5b5f17  
    506506        /* Current values of threads */
    507507        list_foreach(task->threads, th_link, thread_t, thread) {
    508                 irq_spinlock_lock(&thread->lock, false);
    509 
    510508                /* Process only counted threads */
    511509                if (!thread->uncounted) {
    … …  
    515513                        }
    516514
    517                         uret += thread->ucycles;
    518                         kret += thread->kcycles;
     515                        uret += atomic_time_read(&thread->ucycles);
     516                        kret += atomic_time_read(&thread->kcycles);
    519517                }
    520 
    521                 irq_spinlock_unlock(&thread->lock, false);
    522518        }
    523519

  • kernel/generic/src/proc/thread.c

    red7e057 ra5b5f17  
    108108static int threads_cmp(void *, void *);
    109109
    110 /** Thread wrapper.
    111  *
    112  * This wrapper is provided to ensure that every thread makes a call to
    113  * thread_exit() when its implementing function returns.
    114  *
    115  * interrupts_disable() is assumed.
    116  *
    117  */
    118 static void cushion(void)
    119 {
    120         void (*f)(void *) = THREAD->thread_code;
    121         void *arg = THREAD->thread_arg;
    122 
    123         /* This is where each thread wakes up after its creation */
    124         irq_spinlock_unlock(&THREAD->lock, false);
    125         interrupts_enable();
    126 
    127         f(arg);
    128 
    129         thread_exit();
    130 
    131         /* Not reached */
    132 }
    133 
    134110/** Initialization and allocation for thread_t structure
    135111 *
    … …  
    139115        thread_t *thread = (thread_t *) obj;
    140116
    141         irq_spinlock_initialize(&thread->lock, "thread_t_lock");
    142117        link_initialize(&thread->rq_link);
    143118        link_initialize(&thread->wq_link);
    … …  
    221196void thread_wire(thread_t *thread, cpu_t *cpu)
    222197{
    223         irq_spinlock_lock(&thread->lock, true);
    224         thread->cpu = cpu;
     198        ipl_t ipl = interrupts_disable();
     199        atomic_set_unordered(&thread->cpu, cpu);
    225200        thread->nomigrate++;
    226         irq_spinlock_unlock(&thread->lock, true);
     201        interrupts_restore(ipl);
    227202}
    228203
    … …  
    233208void thread_start(thread_t *thread)
    234209{
    235         assert(thread->state == Entering);
    236         thread_ready(thread_ref(thread));
    237 }
    238 
    239 /** Make thread ready
    240  *
    241  * Switch thread to the ready state. Consumes reference passed by the caller.
    242  *
    243  * @param thread Thread to make ready.
    244  *
    245  */
    246 void thread_ready(thread_t *thread)
    247 {
    248         irq_spinlock_lock(&thread->lock, true);
    249 
    250         assert(thread->state != Ready);
    251 
    252         int i = (thread->priority < RQ_COUNT - 1) ?
    253             ++thread->priority : thread->priority;
    254 
    255         /* Prefer the CPU on which the thread ran last */
    256         cpu_t *cpu = thread->cpu ? thread->cpu : CPU;
    257 
    258         thread->state = Ready;
    259 
    260         irq_spinlock_pass(&thread->lock, &(cpu->rq[i].lock));
    261 
    262         /*
    263          * Append thread to respective ready queue
    264          * on respective processor.
    265          */
    266 
    267         list_append(&thread->rq_link, &cpu->rq[i].rq);
    268         cpu->rq[i].n++;
    269         irq_spinlock_unlock(&(cpu->rq[i].lock), true);
    270 
    271         atomic_inc(&nrdy);
    272         atomic_inc(&cpu->nrdy);
     210        assert(atomic_get_unordered(&thread->state) == Entering);
     211        thread_requeue_sleeping(thread_ref(thread));
    273212}
    274213
    … …  
    309248        irq_spinlock_unlock(&tidlock, true);
    310249
    311         context_create(&thread->saved_context, cushion, thread->kstack, STACK_SIZE);
     250        context_create(&thread->saved_context, thread_main_func,
     251            thread->kstack, STACK_SIZE);
    312252
    313253        current_initialize((current_t *) thread->kstack);
    … …  
    317257        thread->thread_code = func;
    318258        thread->thread_arg = arg;
    319         thread->ucycles = 0;
    320         thread->kcycles = 0;
     259        thread->ucycles = ATOMIC_TIME_INITIALIZER();
     260        thread->kcycles = ATOMIC_TIME_INITIALIZER();
    321261        thread->uncounted =
    322262            ((flags & THREAD_FLAG_UNCOUNTED) == THREAD_FLAG_UNCOUNTED);
    323         thread->priority = -1;          /* Start in rq[0] */
    324         thread->cpu = NULL;
     263        atomic_init(&thread->priority, 0);
     264        atomic_init(&thread->cpu, NULL);
    325265        thread->stolen = false;
    326266        thread->uspace =
    … …  
    328268
    329269        thread->nomigrate = 0;
    330         thread->state = Entering;
     270        atomic_init(&thread->state, Entering);
    331271
    332272        atomic_init(&thread->sleep_queue, NULL);
    … …  
    348288#ifdef CONFIG_UDEBUG
    349289        /* Initialize debugging stuff */
    350         thread->btrace = false;
     290        atomic_init(&thread->btrace, false);
    351291        udebug_thread_initialize(&thread->udebug);
    352292#endif
    … …  
    392332
    393333        if (!thread->uncounted) {
    394                 thread->task->ucycles += thread->ucycles;
    395                 thread->task->kcycles += thread->kcycles;
     334                thread->task->ucycles += atomic_time_read(&thread->ucycles);
     335                thread->task->kcycles += atomic_time_read(&thread->kcycles);
    396336        }
    397337
    398338        irq_spinlock_unlock(&thread->task->lock, false);
    399339
    400         assert((thread->state == Exiting) || (thread->state == Lingering));
     340        assert((atomic_get_unordered(&thread->state) == Exiting) || (atomic_get_unordered(&thread->state) == Lingering));
    401341
    402342        /* Clear cpu->fpu_owner if set to this thread. */
    403343#ifdef CONFIG_FPU_LAZY
    404         if (thread->cpu) {
     344        cpu_t *cpu = atomic_get_unordered(&thread->cpu);
     345        if (cpu) {
    405346                /*
    406347                 * We need to lock for this because the old CPU can concurrently try
    … …  
    408349                 * it to finish. An atomic compare-and-swap wouldn't be enough.
    409350                 */
    410                 irq_spinlock_lock(&thread->cpu->fpu_lock, false);
    411 
    412                 thread_t *owner = atomic_load_explicit(&thread->cpu->fpu_owner,
    413                     memory_order_relaxed);
    414 
    415                 if (owner == thread) {
    416                         atomic_store_explicit(&thread->cpu->fpu_owner, NULL,
    417                             memory_order_relaxed);
    418                 }
    419 
    420                 irq_spinlock_unlock(&thread->cpu->fpu_lock, false);
     351                irq_spinlock_lock(&cpu->fpu_lock, false);
     352
     353                if (atomic_get_unordered(&cpu->fpu_owner) == thread)
     354                        atomic_set_unordered(&cpu->fpu_owner, NULL);
     355
     356                irq_spinlock_unlock(&cpu->fpu_lock, false);
    421357        }
    422358#endif
    … …  
    635571                 * the waking thread by the sleeper in thread_wait_finish().
    636572                 */
    637                 thread_ready(thread);
     573                thread_requeue_sleeping(thread);
    638574        }
    639575}
    … …  
    642578void thread_migration_disable(void)
    643579{
     580        ipl_t ipl = interrupts_disable();
     581
    644582        assert(THREAD);
    645 
    646583        THREAD->nomigrate++;
     584
     585        interrupts_restore(ipl);
    647586}
    648587
    … …  
    650589void thread_migration_enable(void)
    651590{
     591        ipl_t ipl = interrupts_disable();
     592
    652593        assert(THREAD);
    653594        assert(THREAD->nomigrate > 0);
    … …  
    655596        if (THREAD->nomigrate > 0)
    656597                THREAD->nomigrate--;
     598
     599        interrupts_restore(ipl);
    657600}
    658601
    … …  
    700643                return EINVAL;
    701644
    702         irq_spinlock_lock(&thread->lock, true);
    703         state_t state = thread->state;
    704         irq_spinlock_unlock(&thread->lock, true);
    705 
    706         errno_t rc = EOK;
    707 
    708         if (state != Exiting)
    709                 rc = _waitq_sleep_timeout(&thread->join_wq, usec, flags);
     645        errno_t rc = _waitq_sleep_timeout(&thread->join_wq, usec, flags);
    710646
    711647        if (rc == EOK)
    … …  
    747683        uint64_t ucycles, kcycles;
    748684        char usuffix, ksuffix;
    749         order_suffix(thread->ucycles, &ucycles, &usuffix);
    750         order_suffix(thread->kcycles, &kcycles, &ksuffix);
     685        order_suffix(atomic_time_read(&thread->ucycles), &ucycles, &usuffix);
     686        order_suffix(atomic_time_read(&thread->kcycles), &kcycles, &ksuffix);
     687
     688        state_t state = atomic_get_unordered(&thread->state);
    751689
    752690        char *name;
    … …  
    762700        else
    763701                printf("%-8" PRIu64 " %-14s %p %-8s %p %-5" PRIu32 "\n",
    764                     thread->tid, name, thread, thread_states[thread->state],
     702                    thread->tid, name, thread, thread_states[state],
    765703                    thread->task, thread->task->container);
    766704
    767705        if (additional) {
    768                 if (thread->cpu)
    769                         printf("%-5u", thread->cpu->id);
     706                cpu_t *cpu = atomic_get_unordered(&thread->cpu);
     707                if (cpu)
     708                        printf("%-5u", cpu->id);
    770709                else
    771710                        printf("none ");
    772711
    773                 if (thread->state == Sleeping) {
     712                if (state == Sleeping) {
    774713                        printf(" %p", thread->sleep_queue);
    775714                }
    … …  
    850789void thread_update_accounting(bool user)
    851790{
     791        assert(interrupts_disabled());
     792
    852793        uint64_t time = get_cycle();
    853794
    854         assert(interrupts_disabled());
    855         assert(irq_spinlock_locked(&THREAD->lock));
    856 
    857795        if (user)
    858                 THREAD->ucycles += time - THREAD->last_cycle;
     796                atomic_time_increment(&THREAD->ucycles, time - THREAD->last_cycle);
    859797        else
    860                 THREAD->kcycles += time - THREAD->last_cycle;
     798                atomic_time_increment(&THREAD->kcycles, time - THREAD->last_cycle);
    861799
    862800        THREAD->last_cycle = time;
    … …  
    969907         */
    970908
    971         irq_spinlock_lock(&thread->lock, true);
    972 
    973         bool sleeping = false;
    974         istate_t *istate = thread->udebug.uspace_state;
    975         if (istate != NULL) {
    976                 printf("Scheduling thread stack trace.\n");
    977                 thread->btrace = true;
    978                 if (thread->state == Sleeping)
    979                         sleeping = true;
    980         } else
    981                 printf("Thread interrupt state not available.\n");
    982 
    983         irq_spinlock_unlock(&thread->lock, true);
    984 
    985         if (sleeping)
    986                 thread_wakeup(thread);
    987 
     909        printf("Scheduling thread stack trace.\n");
     910        atomic_set_unordered(&thread->btrace, true);
     911
     912        thread_wakeup(thread);
    988913        thread_put(thread);
    989914}
    … …  
    10861011                thread_attach(thread, TASK);
    10871012#endif
    1088                 thread_ready(thread);
     1013                thread_start(thread);
     1014                thread_put(thread);
    10891015
    10901016                return 0;

  • kernel/generic/src/syscall/syscall.c

    red7e057 ra5b5f17  
    141141{
    142142        /* Do userpace accounting */
    143         irq_spinlock_lock(&THREAD->lock, true);
     143        ipl_t ipl = interrupts_disable();
    144144        thread_update_accounting(true);
    145         irq_spinlock_unlock(&THREAD->lock, true);
     145        interrupts_restore(ipl);
    146146
    147147#ifdef CONFIG_UDEBUG
    … …  
    191191
    192192        /* Do kernel accounting */
    193         irq_spinlock_lock(&THREAD->lock, true);
     193        ipl = interrupts_disable();
    194194        thread_update_accounting(false);
    195         irq_spinlock_unlock(&THREAD->lock, true);
     195        interrupts_restore(ipl);
    196196
    197197        return rc;

  • kernel/generic/src/sysinfo/stats.c

    red7e057 ra5b5f17  
    299299{
    300300        assert(interrupts_disabled());
    301         assert(irq_spinlock_locked(&thread->lock));
    302301
    303302        stats_thread->thread_id = thread->tid;
    304303        stats_thread->task_id = thread->task->taskid;
    305         stats_thread->state = thread->state;
    306         stats_thread->priority = thread->priority;
    307         stats_thread->ucycles = thread->ucycles;
    308         stats_thread->kcycles = thread->kcycles;
    309 
    310         if (thread->cpu != NULL) {
     304        stats_thread->state = atomic_get_unordered(&thread->state);
     305        stats_thread->priority = atomic_get_unordered(&thread->priority);
     306        stats_thread->ucycles = atomic_time_read(&thread->ucycles);
     307        stats_thread->kcycles = atomic_time_read(&thread->kcycles);
     308
     309        cpu_t *cpu = atomic_get_unordered(&thread->cpu);
     310
     311        if (cpu != NULL) {
    311312                stats_thread->on_cpu = true;
    312                 stats_thread->cpu = thread->cpu->id;
     313                stats_thread->cpu = cpu->id;
    313314        } else
    314315                stats_thread->on_cpu = false;
    … …  
    361362        thread_t *thread = thread_first();
    362363        while (thread != NULL) {
    363                 /* Interrupts are already disabled */
    364                 irq_spinlock_lock(&thread->lock, false);
    365 
    366364                /* Record the statistics and increment the index */
    367365                produce_stats_thread(thread, &stats_threads[i]);
    368366                i++;
    369 
    370                 irq_spinlock_unlock(&thread->lock, false);
    371367
    372368                thread = thread_next(thread);
    … …  
    624620                ret.data.size = sizeof(stats_thread_t);
    625621
    626                 /*
    627                  * Replaced hand-over-hand locking with regular nested sections
    628                  * to avoid weak reference leak issues.
    629                  */
    630                 irq_spinlock_lock(&thread->lock, false);
    631622                produce_stats_thread(thread, stats_thread);
    632                 irq_spinlock_unlock(&thread->lock, false);
    633623
    634624                irq_spinlock_unlock(&threads_lock, true);

  • kernel/generic/src/time/clock.c

    red7e057 ra5b5f17  
    123123static void cpu_update_accounting(void)
    124124{
     125        // FIXME: get_cycle() is unimplemented on several platforms
    125126        uint64_t now = get_cycle();
    126127        atomic_time_increment(&CPU->busy_cycles, now - CPU_LOCAL->last_cycle);

  • kernel/generic/src/udebug/udebug_ops.c

    red7e057 ra5b5f17  
    9090        }
    9191
    92         irq_spinlock_lock(&thread->lock, true);
    93 
    9492        /* Verify that 'thread' is a userspace thread. */
    9593        if (!thread->uspace) {
    96                 /* It's not, deny its existence */
    97                 irq_spinlock_unlock(&thread->lock, true);
    9894                mutex_unlock(&TASK->udebug.lock);
    9995                return ENOENT;
    10096        }
    101 
    102         /* Verify debugging state. */
    103         if (thread->udebug.active != true) {
    104                 /* Not in debugging session or undesired GO state */
    105                 irq_spinlock_unlock(&thread->lock, true);
    106                 mutex_unlock(&TASK->udebug.lock);
    107                 return ENOENT;
    108         }
    109 
    110         /* Now verify that the thread belongs to the current task. */
    111         if (thread->task != TASK) {
    112                 /* No such thread belonging this task */
    113                 irq_spinlock_unlock(&thread->lock, true);
    114                 mutex_unlock(&TASK->udebug.lock);
    115                 return ENOENT;
    116         }
    117 
    118         irq_spinlock_unlock(&thread->lock, true);
    119 
    120         /* Only mutex TASK->udebug.lock left. */
    12197
    12298        /*
    … …  
    126102         */
    127103        mutex_lock(&thread->udebug.lock);
     104
     105        /* Verify debugging state. */
     106        if (thread->udebug.active != true) {
     107                /* Not in debugging session or undesired GO state */
     108                mutex_unlock(&thread->udebug.lock);
     109                mutex_unlock(&TASK->udebug.lock);
     110                return ENOENT;
     111        }
     112
     113        /* Now verify that the thread belongs to the current task. */
     114        if (thread->task != TASK) {
     115                /* No such thread belonging this task */
     116                mutex_unlock(&thread->udebug.lock);
     117                mutex_unlock(&TASK->udebug.lock);
     118                return ENOENT;
     119        }
    128120
    129121        /* The big task mutex is no longer needed. */
    … …  
    388380        /* FIXME: make sure the thread isn't past debug shutdown... */
    389381        list_foreach(TASK->threads, th_link, thread_t, thread) {
    390                 irq_spinlock_lock(&thread->lock, false);
    391382                bool uspace = thread->uspace;
    392                 irq_spinlock_unlock(&thread->lock, false);
    393383
    394384                /* Not interested in kernel threads. */
Note: See TracChangeset for help on using the changeset viewer.