openvswitch/lib/ovs-atomic-gcc4+.h

/*
 * Copyright (c) 2013, 2014 Nicira, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/* This header implements atomic operation primitives on GCC 4.x. */
#ifndef IN_OVS_ATOMIC_H
#error "This header should only be included indirectly via ovs-atomic.h."
#endif

#include "ovs-atomic-locked.h"
#define OVS_ATOMIC_GCC4P_IMPL 1

#define ATOMIC(TYPE) TYPE

#define ATOMIC_BOOL_LOCK_FREE 2
#define ATOMIC_CHAR_LOCK_FREE 2
#define ATOMIC_SHORT_LOCK_FREE 2
#define ATOMIC_INT_LOCK_FREE 2
#define ATOMIC_LONG_LOCK_FREE (ULONG_MAX <= UINTPTR_MAX ? 2 : 0)
#define ATOMIC_LLONG_LOCK_FREE (ULLONG_MAX <= UINTPTR_MAX ? 2 : 0)
#define ATOMIC_POINTER_LOCK_FREE 2

typedef enum {
    memory_order_relaxed,
    memory_order_consume,
    memory_order_acquire,
    memory_order_release,
    memory_order_acq_rel,
    memory_order_seq_cst
} memory_order;

#define IS_LOCKLESS_ATOMIC(OBJECT) (sizeof(OBJECT) <= sizeof(void *))

#define ATOMIC_VAR_INIT(VALUE) VALUE
#define atomic_init(OBJECT, VALUE) (*(OBJECT) = (VALUE), (void) 0)

static inline void
atomic_thread_fence(memory_order order)
{
    if (order != memory_order_relaxed) {
        __sync_synchronize();
    }
}

static inline void
atomic_thread_fence_if_seq_cst(memory_order order)
{
    if (order == memory_order_seq_cst) {
        __sync_synchronize();
    }
}

static inline void
atomic_signal_fence(memory_order order)
{
    if (order != memory_order_relaxed) {
        asm volatile("" : : : "memory");
    }
}

#define atomic_is_lock_free(OBJ)                \
    ((void) *(OBJ),                             \
     IS_LOCKLESS_ATOMIC(*(OBJ)) ? 2 : 0)

#define atomic_store(DST, SRC) \
    atomic_store_explicit(DST, SRC, memory_order_seq_cst)
#define atomic_store_explicit(DST, SRC, ORDER)          \
    ({                                                  \
        typeof(DST) dst__ = (DST);                      \
        typeof(SRC) src__ = (SRC);                      \
                                                        \
        if (IS_LOCKLESS_ATOMIC(*dst__)) {               \
            atomic_thread_fence(ORDER);                 \
            *(typeof(*(DST)) volatile *)dst__ = src__;  \
            atomic_thread_fence_if_seq_cst(ORDER);      \
        } else {                                        \
            atomic_store_locked(dst__, src__);          \
        }                                               \
        (void) 0;                                       \
    })
#define atomic_read(SRC, DST) \
    atomic_read_explicit(SRC, DST, memory_order_seq_cst)
#define atomic_read_explicit(SRC, DST, ORDER)           \
    ({                                                  \
        typeof(DST) dst__ = (DST);                      \
        typeof(SRC) src__ = (SRC);                      \
                                                        \
        if (IS_LOCKLESS_ATOMIC(*src__)) {               \
            atomic_thread_fence_if_seq_cst(ORDER);      \
            *dst__ = *(typeof(*(SRC)) volatile *)src__; \
        } else {                                        \
            atomic_read_locked(src__, dst__);           \
        }                                               \
        (void) 0;                                       \
    })

#define atomic_compare_exchange_strong(DST, EXP, SRC)   \
    ({                                                  \
        typeof(DST) dst__ = (DST);                      \
        typeof(EXP) expp__ = (EXP);                     \
        typeof(SRC) src__ = (SRC);                      \
        typeof(SRC) exp__ = *expp__;                    \
        typeof(SRC) ret__;                              \
                                                        \
        ret__ = __sync_val_compare_and_swap(dst__, exp__, src__); \
        if (ret__ != exp__) {                                     \
            *expp__ = ret__;                                      \
        }                                                         \
        ret__ == exp__;                                           \
    })
#define atomic_compare_exchange_strong_explicit(DST, EXP, SRC, ORD1, ORD2) \
    ((void) (ORD1), (void) (ORD2), \
     atomic_compare_exchange_strong(DST, EXP, SRC))
#define atomic_compare_exchange_weak            \
    atomic_compare_exchange_strong
#define atomic_compare_exchange_weak_explicit   \
    atomic_compare_exchange_strong_explicit

#define atomic_op__(RMW, OP, ARG, ORIG)                     \
    ({                                                      \
        typeof(RMW) rmw__ = (RMW);                          \
        typeof(ARG) arg__ = (ARG);                          \
        typeof(ORIG) orig__ = (ORIG);                       \
                                                            \
        if (IS_LOCKLESS_ATOMIC(*rmw__)) {                   \
            *orig__ = __sync_fetch_and_##OP(rmw__, arg__);  \
        } else {                                            \
            atomic_op_locked(rmw__, OP, arg__, orig__);     \
        }                                                   \
        (void) 0;                                           \
    })

#define atomic_add(RMW, ARG, ORIG) atomic_op__(RMW, add, ARG, ORIG)
#define atomic_sub(RMW, ARG, ORIG) atomic_op__(RMW, sub, ARG, ORIG)
#define atomic_or(RMW, ARG, ORIG) atomic_op__(RMW, or,  ARG, ORIG)
#define atomic_xor(RMW, ARG, ORIG) atomic_op__(RMW, xor, ARG, ORIG)
#define atomic_and(RMW, ARG, ORIG) atomic_op__(RMW, and, ARG, ORIG)

#define atomic_add_explicit(RMW, OPERAND, ORIG, ORDER)  \
    ((void) (ORDER), atomic_add(RMW, OPERAND, ORIG))
#define atomic_sub_explicit(RMW, OPERAND, ORIG, ORDER)  \
    ((void) (ORDER), atomic_sub(RMW, OPERAND, ORIG))
#define atomic_or_explicit(RMW, OPERAND, ORIG, ORDER)   \
    ((void) (ORDER), atomic_or(RMW, OPERAND, ORIG))
#define atomic_xor_explicit(RMW, OPERAND, ORIG, ORDER)  \
    ((void) (ORDER), atomic_xor(RMW, OPERAND, ORIG))
#define atomic_and_explicit(RMW, OPERAND, ORIG, ORDER)  \
    ((void) (ORDER), atomic_and(RMW, OPERAND, ORIG))

/* atomic_flag */

typedef struct {
    int b;
} atomic_flag;
#define ATOMIC_FLAG_INIT { false }

static inline bool
atomic_flag_test_and_set_explicit(volatile atomic_flag *object,
                                  memory_order order)
{
    bool old;

    /* __sync_lock_test_and_set() by itself is an acquire barrier.
     * For anything higher additional barriers are needed. */
    if (order > memory_order_acquire) {
        atomic_thread_fence(order);
    }
    old = __sync_lock_test_and_set(&object->b, 1);
    atomic_thread_fence_if_seq_cst(order);

    return old;
}

#define atomic_flag_test_and_set(FLAG)                                  \
    atomic_flag_test_and_set_explicit(FLAG, memory_order_seq_cst)

static inline void
atomic_flag_clear_explicit(volatile atomic_flag *object,
                           memory_order order)
{
    /* __sync_lock_release() by itself is a release barrier.  For
     * anything else additional barrier may be needed. */
    if (order != memory_order_release) {
        atomic_thread_fence(order);
    }
    __sync_lock_release(&object->b);
    atomic_thread_fence_if_seq_cst(order);
}

#define atomic_flag_clear(FLAG)                                 \
    atomic_flag_clear_explicit(FLAG, memory_order_seq_cst)