From 41b1c6fa42d3a8a34ae39155eef09a01219eeab5 Mon Sep 17 00:00:00 2001 From: Jinkun Jang Date: Sat, 16 Mar 2013 01:25:41 +0900 Subject: [PATCH] merge with master --- AUTHORS | 0 COPYING | 28 + ChangeLog | 0 Makefile.am | 7 + NEWS | 0 README | 0 autogen.sh | 12 + configure.ac | 30 + libwsbm.pc.in | 10 + packaging/0001-CDV-include-tweaks.patch | 53 ++ packaging/libwsbm.changes | 62 ++ packaging/libwsbm.patch | 207 ++++++ packaging/libwsbm.spec | 67 ++ src/Makefile.am | 27 + src/wsbm_atomic.h | 76 ++ src/wsbm_driver.c | 243 ++++++ src/wsbm_driver.h | 131 ++++ src/wsbm_fencemgr.c | 474 ++++++++++++ src/wsbm_fencemgr.h | 162 ++++ src/wsbm_mallocpool.c | 187 +++++ src/wsbm_manager.c | 1225 +++++++++++++++++++++++++++++++ src/wsbm_manager.h | 182 +++++ src/wsbm_mm.c | 289 ++++++++ src/wsbm_mm.h | 75 ++ src/wsbm_pool.h | 161 ++++ src/wsbm_priv.h | 45 ++ src/wsbm_slabpool.c | 1203 ++++++++++++++++++++++++++++++ src/wsbm_ttmpool.c | 509 +++++++++++++ src/wsbm_userpool.c | 689 +++++++++++++++++ src/wsbm_util.h | 76 ++ 30 files changed, 6230 insertions(+) create mode 100644 AUTHORS create mode 100644 COPYING create mode 100644 ChangeLog create mode 100644 Makefile.am create mode 100644 NEWS create mode 100644 README create mode 100755 autogen.sh create mode 100644 configure.ac create mode 100644 libwsbm.pc.in create mode 100644 packaging/0001-CDV-include-tweaks.patch create mode 100644 packaging/libwsbm.changes create mode 100644 packaging/libwsbm.patch create mode 100644 packaging/libwsbm.spec create mode 100644 src/Makefile.am create mode 100644 src/wsbm_atomic.h create mode 100644 src/wsbm_driver.c create mode 100644 src/wsbm_driver.h create mode 100644 src/wsbm_fencemgr.c create mode 100644 src/wsbm_fencemgr.h create mode 100644 src/wsbm_mallocpool.c create mode 100644 src/wsbm_manager.c create mode 100644 src/wsbm_manager.h create mode 100644 src/wsbm_mm.c create mode 100644 src/wsbm_mm.h create mode 100644 src/wsbm_pool.h create mode 100644 src/wsbm_priv.h create mode 100644 src/wsbm_slabpool.c create mode 100644 src/wsbm_ttmpool.c create mode 100644 src/wsbm_userpool.c create mode 100644 src/wsbm_util.h diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..e69de29 diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..362bf8e --- /dev/null +++ b/COPYING @@ -0,0 +1,28 @@ +/************************************************************************** + * + * Copyright 2006-2008 Tungsten Graphics, Inc., Cedar Park, TX., USA + * All Rights Reserved. + * Copyright 2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ diff --git a/ChangeLog b/ChangeLog new file mode 100644 index 0000000..e69de29 diff --git a/Makefile.am b/Makefile.am new file mode 100644 index 0000000..2940c78 --- /dev/null +++ b/Makefile.am @@ -0,0 +1,7 @@ +SUBDIRS = src +EXTRA_DIST = COPYING NEWS README libwsbm.pc.in + +pkgconfigdir = @pkgconfigdir@ +pkgconfig_DATA = libwsbm.pc + + diff --git a/NEWS b/NEWS new file mode 100644 index 0000000..e69de29 diff --git a/README b/README new file mode 100644 index 0000000..e69de29 diff --git a/autogen.sh b/autogen.sh new file mode 100755 index 0000000..904cd67 --- /dev/null +++ b/autogen.sh @@ -0,0 +1,12 @@ +#! /bin/sh + +srcdir=`dirname $0` +test -z "$srcdir" && srcdir=. + +ORIGDIR=`pwd` +cd $srcdir + +autoreconf -v --install || exit 1 +cd $ORIGDIR || exit $? + +$srcdir/configure --enable-maintainer-mode "$@" diff --git a/configure.ac b/configure.ac new file mode 100644 index 0000000..24191f2 --- /dev/null +++ b/configure.ac @@ -0,0 +1,30 @@ +AC_PREREQ(2.57) +AC_INIT([libwsbm], 1.1.0, [thomas@tungstengraphics.com], libwsbm) +AC_CONFIG_SRCDIR([Makefile.am]) +AM_INIT_AUTOMAKE([dist-bzip2]) + +AM_CONFIG_HEADER([config.h]) + +AC_DISABLE_STATIC +AC_PROG_LIBTOOL +AC_PROG_CC +PKG_PROG_PKG_CONFIG + +PKG_CHECK_MODULES(libdrm, libdrm) +AC_SUBST(libdrm_CFLAGS) +AC_SUBST(libdrm_LIBS) +AC_HEADER_STDC +AC_SYS_LARGEFILE + +AC_CHECK_HEADER(pthread.h, [ + AC_SEARCH_LIBS(pthread_cond_init, pthread, + [AC_DEFINE(HAVE_PTHREADS, 1, "os has pthreads")],,,) + ],,,) + +pkgconfigdir=${libdir}/pkgconfig +AC_SUBST(pkgconfigdir) + +AC_OUTPUT([ + Makefile + src/Makefile + libwsbm.pc]) diff --git a/libwsbm.pc.in b/libwsbm.pc.in new file mode 100644 index 0000000..dfff843 --- /dev/null +++ b/libwsbm.pc.in @@ -0,0 +1,10 @@ +prefix=@prefix@ +exec_prefix=@exec_prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: libwsbm +Description: Buffer manager abstraction library. +Version: @PACKAGE_VERSION@ +Libs: -L${libdir} -lwsbm +Cflags: -I${includedir}/wsbm diff --git a/packaging/0001-CDV-include-tweaks.patch b/packaging/0001-CDV-include-tweaks.patch new file mode 100644 index 0000000..a7088e5 --- /dev/null +++ b/packaging/0001-CDV-include-tweaks.patch @@ -0,0 +1,53 @@ +From cff48cf178ec93de94b03119bc28b719c75bedbf Mon Sep 17 00:00:00 2001 +From: Joe Konno +Date: Mon, 15 Aug 2011 11:34:58 -0700 +Subject: [PATCH] CDV include tweaks + +--- + src/wsbm_fencemgr.c | 2 +- + src/wsbm_slabpool.c | 2 +- + src/wsbm_ttmpool.c | 2 +- + 3 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/src/wsbm_fencemgr.c b/src/wsbm_fencemgr.c +index 0e7a302..4010ac9 100644 +--- a/src/wsbm_fencemgr.c ++++ b/src/wsbm_fencemgr.c +@@ -38,7 +38,7 @@ + #include "wsbm_pool.h" + #include "wsbm_manager.h" + #include +-#include ++#include + #include + #include + +diff --git a/src/wsbm_slabpool.c b/src/wsbm_slabpool.c +index 51a85d0..56f7d65 100644 +--- a/src/wsbm_slabpool.c ++++ b/src/wsbm_slabpool.c +@@ -34,7 +34,7 @@ + #include "config.h" + #endif + +-#include ++#include + #include + #include + #include +diff --git a/src/wsbm_ttmpool.c b/src/wsbm_ttmpool.c +index 5867428..93da32a 100644 +--- a/src/wsbm_ttmpool.c ++++ b/src/wsbm_ttmpool.c +@@ -42,7 +42,7 @@ + #include "assert.h" + #include "wsbm_priv.h" + #include "wsbm_manager.h" +-#include "ttm/ttm_placement_user.h" ++#include + + #define DRMRESTARTCOMMANDWRITE(_fd, _val, _arg, _ret) \ + do { \ +-- +1.7.4.4 + diff --git a/packaging/libwsbm.changes b/packaging/libwsbm.changes new file mode 100644 index 0000000..8621a69 --- /dev/null +++ b/packaging/libwsbm.changes @@ -0,0 +1,62 @@ +* Thu Jun 14 2012 vivian zhang - 1.1.0 +- Modified source directory name in spec file to fix the build failure + +* Wed Jun 13 2012 vivian zhang - 1.1.0 +- Import the package for Tizen: Buffer manager abstraction library + +* Fri Jun 01 2012 vivian zhang - 1.1.0 +- Initial import for Tizen + +* Tue Jan 24 2012 Markus Lehtonen - 1.1.0 +- Back to orinal naming, remove the cdv postfix + +* Mon Aug 15 2011 Joe Konno - 1.1.0-3 +- After consulting with developers, placed PSB headers in cdv-headers +- Modified source #include statements accordingly + +* Wed Aug 10 2011 Joe Konno - 1.1.0-2 +- Re-packaged as 'libwsbm-cdv' +- Per kernelnewbies.org, placed PSB kernel headers in source tree +- Building against PSB headers from kernel-adaptation-pc 3.0 CDV + +* Wed Jun 30 2010 chris.e.ferron@linux.intel.com - 1.1.0 +- added patch to fix memory leak for EXA under gnome for powerVR driver sharebuffer issue. + +* Thu Apr 29 2010 Anas Nashif - 1.1.0 +- Make it exclusive to %ix86 + +* Mon Aug 24 2009 Priya Vijayan 1.1.0 +- Update to 1.1.0 beta + +* Sun Aug 16 2009 Anas Nashif - 1.1.0_20090609 +- Use _ instead of ~ in version + +* Tue Jun 16 2009 Priya Vijayan 1.1.0~20090318 +- Update to 1.1.0-1.0.7.20090609 + +* Tue Apr 14 2009 Anas Nashif 1.1.0~20090318 +- Added missing required packagein -devel + +* Fri Mar 27 2009 Anas Nashif 1.1.0~20090318 +- Fixed BuildRequires, now use psb-headers + +* Fri Mar 27 2009 Anas Nashif 1.1.0~20090318 +- Fixed Requires + +* Wed Feb 25 2009 bjhueni 1.0.5 +- request package merge to MRST. Build & install testing completed + +* Fri Jan 16 2009 Priya Vijayan 1.0.2 +- Updating source + +* Tue Jan 06 2009 Priya Vijayan 1.0.1 +- fixed spec + +* Tue Jan 06 2009 Priya Vijayan 1.0.0 +- Update to 1.0.1-working combination with X Server + +* Thu Nov 27 2008 Yin Kangkai +- actually compress the tar ball to bz2 format + +* Nov 2008 Priya Vijayan +- Version 0.28 diff --git a/packaging/libwsbm.patch b/packaging/libwsbm.patch new file mode 100644 index 0000000..6f3a109 --- /dev/null +++ b/packaging/libwsbm.patch @@ -0,0 +1,207 @@ +diff -uBr libwsbm-old/src/wsbm_manager.c libwsbm/src/wsbm_manager.c +--- libwsbm-old/src/wsbm_manager.c 2009-08-20 12:09:37.000000000 -0700 ++++ libwsbm/src/wsbm_manager.c 2010-06-25 19:03:58.305282722 -0700 +@@ -610,6 +610,139 @@ + return retval; + } + ++int ++wsbmBODataUB(struct _WsbmBufferObject *buf, ++ unsigned size, const void *data, ++ struct _WsbmBufferPool *newPool, uint32_t placement, const unsigned long *user_ptr) ++{ ++ void *virtual = NULL; ++ int newBuffer; ++ int retval = 0; ++ struct _WsbmBufStorage *storage; ++ int synced = 0; ++ uint32_t placement_diff; ++ struct _WsbmBufferPool *curPool; ++ extern struct _WsbmBufStorage * ++ttm_pool_ub_create(struct _WsbmBufferPool *pool, ++ unsigned long size, uint32_t placement, unsigned alignment, const unsigned long *user_ptr); ++ ++ if (buf->bufferType == WSBM_BUFFER_SIMPLE) ++ return -EINVAL; ++ ++ storage = buf->storage; ++ ++ if (newPool == NULL) ++ newPool = buf->pool; ++ ++ if (newPool == NULL) ++ return -EINVAL; ++ ++ newBuffer = (!storage || storage->pool != newPool || ++ storage->pool->size(storage) < size || ++ storage->pool->size(storage) > ++ size + WSBM_BODATA_SIZE_ACCEPT); ++ ++ if (!placement) ++ placement = buf->placement; ++ ++ if (newBuffer) { ++ if (buf->bufferType == WSBM_BUFFER_REF) ++ return -EINVAL; ++ ++ wsbmBufStorageUnref(&buf->storage); ++ ++ if (size == 0) { ++ buf->pool = newPool; ++ buf->placement = placement; ++ retval = 0; ++ goto out; ++ } ++ ++ buf->storage = ++ //newPool->create(newPool, size, placement, buf->alignment); ++ ttm_pool_ub_create(newPool, size, placement, buf->alignment, user_ptr); ++ if (!buf->storage) { ++ retval = -ENOMEM; ++ goto out; ++ } ++ ++ buf->placement = placement; ++ buf->pool = newPool; ++ } else if (wsbmAtomicRead(&storage->onList) || ++ 0 != storage->pool->syncforcpu(storage, WSBM_SYNCCPU_WRITE | ++ WSBM_SYNCCPU_DONT_BLOCK)) { ++ /* ++ * Buffer is busy. need to create a new one. ++ */ ++ ++ struct _WsbmBufStorage *tmp_storage; ++ ++ curPool = storage->pool; ++ ++ tmp_storage = ++ curPool->create(curPool, size, placement, buf->alignment); ++ ++ if (tmp_storage) { ++ wsbmBufStorageUnref(&buf->storage); ++ buf->storage = tmp_storage; ++ buf->placement = placement; ++ } else { ++ retval = curPool->syncforcpu(storage, WSBM_SYNCCPU_WRITE); ++ if (retval) ++ goto out; ++ synced = 1; ++ } ++ } else ++ synced = 1; ++ ++ placement_diff = placement ^ buf->placement; ++ ++ /* ++ * We might need to change buffer placement. ++ */ ++ ++ storage = buf->storage; ++ curPool = storage->pool; ++ ++ if (placement_diff) { ++ assert(curPool->setStatus != NULL); ++ curPool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE); ++ retval = curPool->setStatus(storage, ++ placement_diff & placement, ++ placement_diff & ~placement); ++ if (retval) ++ goto out; ++ ++ buf->placement = placement; ++ ++ } ++ ++ if (!synced) { ++ retval = curPool->syncforcpu(buf->storage, WSBM_SYNCCPU_WRITE); ++ ++ if (retval) ++ goto out; ++ synced = 1; ++ } ++ ++ storage = buf->storage; ++ curPool = storage->pool; ++ ++ if (data) { ++ retval = curPool->map(storage, WSBM_ACCESS_WRITE, &virtual); ++ if (retval) ++ goto out; ++ memcpy(virtual, data, size); ++ curPool->unmap(storage); ++ } ++ ++ out: ++ ++ if (synced) ++ curPool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE); ++ ++ return retval; ++} + static struct _WsbmBufStorage * + wsbmStorageClone(struct _WsbmBufferObject *buf) + { +diff -uBr libwsbm-old/src/wsbm_ttmpool.c libwsbm/src/wsbm_ttmpool.c +--- libwsbm-old/src/wsbm_ttmpool.c 2009-08-20 12:09:37.000000000 -0700 ++++ libwsbm/src/wsbm_ttmpool.c 2010-06-25 19:03:58.304282858 -0700 +@@ -507,3 +507,60 @@ + pool->setStatus = &pool_setStatus; + return pool; + } ++ ++ ++struct _WsbmBufStorage * ++ttm_pool_ub_create(struct _WsbmBufferPool *pool, ++ unsigned long size, uint32_t placement, unsigned alignment, const unsigned long *user_ptr) ++{ ++ struct _TTMBuffer *dBuf = (struct _TTMBuffer *) ++ calloc(1, sizeof(*dBuf)); ++ struct _TTMPool *ttmPool = containerOf(pool, struct _TTMPool, pool); ++ int ret; ++ unsigned pageSize = ttmPool->pageSize; ++ union ttm_pl_create_ub_arg arg; ++ ++ if (!dBuf) ++ return NULL; ++ ++ if ((alignment > pageSize) && (alignment % pageSize)) ++ goto out_err0; ++ ++ ret = wsbmBufStorageInit(&dBuf->buf, pool); ++ if (ret) ++ goto out_err0; ++ ++ ret = WSBM_COND_INIT(&dBuf->event); ++ if (ret) ++ goto out_err1; ++ ++ arg.req.size = size; ++ arg.req.placement = placement; ++ arg.req.page_alignment = alignment / pageSize; ++ arg.req.user_address = user_ptr; ++ ++ DRMRESTARTCOMMANDWRITEREAD(pool->fd, ttmPool->devOffset + TTM_PL_CREATE_UB, ++ arg, ret); ++ ++ if (ret) ++ goto out_err2; ++ ++ dBuf->requestedSize = size; ++ dBuf->kBuf.gpuOffset = arg.rep.gpu_offset; ++ dBuf->mapHandle = arg.rep.map_handle; ++ dBuf->realSize = arg.rep.bo_size; ++ dBuf->kBuf.placement = arg.rep.placement; ++ dBuf->kBuf.handle = arg.rep.handle; ++ ++ return &dBuf->buf; ++ ++ out_err2: ++ WSBM_COND_FREE(&dBuf->event); ++ out_err1: ++ wsbmBufStorageTakedown(&dBuf->buf); ++ out_err0: ++ free(dBuf); ++ return NULL; ++} ++ ++ diff --git a/packaging/libwsbm.spec b/packaging/libwsbm.spec new file mode 100644 index 0000000..0a634af --- /dev/null +++ b/packaging/libwsbm.spec @@ -0,0 +1,67 @@ +Summary: Buffer manager abstraction library +Name: libwsbm +Version: 1.1.0 +Release: 1 +Group: Development/Libraries +License: MIT +Source: libwsbm-src.1.1.0.tar.bz2 +Patch0: libwsbm.patch +Patch1: 0001-CDV-include-tweaks.patch +URL: http://www.x.org/wiki/libwsbm +BuildRequires: libdrm-devel +BuildRequires: psb-headers +Conflicts: libwsbm-cdv +ExclusiveArch: %ix86 + +%description +The libwsbm (wsbm stands for Window System Buffer Manager) library previously +lived as a dri buffer manager within mesa. This package builds against +PVR-provided PSB headers found in the kernel-adaptation-pc image. This package +ought only be installed on a CDV system. + +%package devel +Summary: Development headers for libwsbm +Group: Development/Libraries +Requires: %{name} = %{version}-%{release} + +%description devel +Development headers (Graphics) for libwsbm + +%prep +%setup -q -n libwsbm-src.%{version} + +# libswbm.patch +%patch0 -p1 +# 0001-CDV-include-tweaks.patch +%patch1 -p1 + +%build +%autogen +make %{?_smp_mflags} + + +%install +rm -rf $RPM_BUILD_ROOT +%make_install + + +%clean +rm -rf $RPM_BUILD_ROOT + +%post -p /sbin/ldconfig +%postun -p /sbin/ldconfig + +%files +%defattr(-,root,root,-) +%{_libdir}/libwsbm.so.1 +%{_libdir}/libwsbm.so.1.1.0 + + +%files devel +%defattr(-,root,root,-) +%{_libdir}/pkgconfig/libwsbm.pc +%{_libdir}/libwsbm.so +%{_includedir}/wsbm/*.h + + +%changelog diff --git a/src/Makefile.am b/src/Makefile.am new file mode 100644 index 0000000..4778f37 --- /dev/null +++ b/src/Makefile.am @@ -0,0 +1,27 @@ + +lib_LTLIBRARIES = libwsbm.la + +libwsbm_la_CFLAGS = @libdrm_CFLAGS@ -Wall +libwsbm_la_LDFLAGS = -version-number 1:1:0 -no-undefined @libdrm_LIBS@ +libwsbm_la_SOURCES = \ + wsbm_fencemgr.c \ + wsbm_fencemgr.h \ + wsbm_manager.c \ + wsbm_manager.h \ + wsbm_mm.c \ + wsbm_mm.h \ + wsbm_pool.h \ + wsbm_util.h \ + wsbm_mallocpool.c \ + wsbm_driver.h \ + wsbm_driver.c \ + wsbm_ttmpool.c \ + wsbm_slabpool.c \ + wsbm_userpool.c \ + wsbm_priv.h + + +libwsbmincludedir = ${includedir}/wsbm +libwsbminclude_HEADERS = wsbm_manager.h wsbm_pool.h wsbm_driver.h \ + wsbm_fencemgr.h wsbm_util.h wsbm_atomic.h + diff --git a/src/wsbm_atomic.h b/src/wsbm_atomic.h new file mode 100644 index 0000000..7be2da9 --- /dev/null +++ b/src/wsbm_atomic.h @@ -0,0 +1,76 @@ +/** + * This file is under no copyright claims due to its + * simplicity. + */ + +#ifndef _WSBM_ATOMIC_H_ +#define _WSBM_ATOMIC_H_ + +#include + +struct _WsbmAtomic +{ + int32_t count; +}; + +#define wsbmAtomicInit(_i) {(i)} +#define wsbmAtomicSet(_v, _i) (((_v)->count) = (_i)) +#define wsbmAtomicRead(_v) ((_v)->count) + +static inline int +wsbmAtomicIncZero(struct _WsbmAtomic *v) +{ + unsigned char c; + __asm__ __volatile__("lock; incl %0; sete %1":"+m"(v->count), "=qm"(c) + ::"memory"); + + return c != 0; +} + +static inline int +wsbmAtomicDecNegative(struct _WsbmAtomic *v) +{ + unsigned char c; + int i = -1; + __asm__ __volatile__("lock; addl %2,%0; sets %1":"+m"(v->count), "=qm"(c) + :"ir"(i):"memory"); + + return c; +} + +static inline int +wsbmAtomicDecZero(struct _WsbmAtomic *v) +{ + unsigned char c; + + __asm__ __volatile__("lock; decl %0; sete %1":"+m"(v->count), "=qm"(c) + ::"memory"); + + return c != 0; +} + +static inline void +wsbmAtomicInc(struct _WsbmAtomic *v) +{ + __asm__ __volatile__("lock; incl %0":"+m"(v->count)); +} + +static inline void +wsbmAtomicDec(struct _WsbmAtomic *v) +{ + __asm__ __volatile__("lock; decl %0":"+m"(v->count)); +} + +static inline int32_t +wsbmAtomicCmpXchg(volatile struct _WsbmAtomic *v, int32_t old, int32_t new) +{ + int32_t previous; + + __asm__ __volatile__("lock; cmpxchgl %k1,%2":"=a"(previous) + :"r"(new), "m"(v->count), "0"(old) + :"memory"); + + return previous; +} + +#endif diff --git a/src/wsbm_driver.c b/src/wsbm_driver.c new file mode 100644 index 0000000..98f846d --- /dev/null +++ b/src/wsbm_driver.c @@ -0,0 +1,243 @@ +/************************************************************************** + * + * Copyright 2008 Tungsten Graphics, Inc., Cedar Park, TX., USA + * All Rights Reserved. + * Copyright 2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: + * Thomas Hellström + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include "wsbm_driver.h" + +struct _WsbmThreadFuncs *wsbmCurThreadFunc = NULL; +struct _WsbmVNodeFuncs *wsbmCurVNodeFunc = NULL; + +/* + * Single-threaded implementation. + */ + +static int +n_mutexInit(struct _WsbmMutex *mutex, struct _WsbmThreadFuncs *func) +{ + mutex->func = func; + return 0; +} + +static int +n_condInit(struct _WsbmCond *cond, struct _WsbmThreadFuncs *func) +{ + cond->func = func; + return 0; +} + +static void +n_mutexNone(struct _WsbmMutex *mutex) +{ + ; +} + +static void +n_condNone(struct _WsbmCond *cond) +{ + ; +} + +static void +n_condWait(struct _WsbmCond *cond, struct _WsbmMutex *mutex) +{ + ; +} + +static struct _WsbmThreadFuncs nullFunc = { + .mutexInit = n_mutexInit, + .mutexFree = n_mutexNone, + .mutexLock = n_mutexNone, + .mutexUnlock = n_mutexNone, + .condInit = n_condInit, + .condFree = n_condNone, + .condWait = n_condWait, + .condBroadcast = n_condNone +}; + +struct _WsbmThreadFuncs * +wsbmNullThreadFuncs(void) +{ + return &nullFunc; +} + +#if (HAVE_PTHREADS == 1) +#include "pthread.h" + +/* + * pthreads implementation: + */ + +struct _WsbmPMutex +{ + struct _WsbmThreadFuncs *func; + pthread_mutex_t mutex; +}; + +struct _WsbmPCond +{ + struct _WsbmThreadFuncs *func; + pthread_cond_t cond; +}; + +static inline struct _WsbmPMutex * +pMutexConvert(struct _WsbmMutex *m) +{ + union _PMutexConverter + { + struct _WsbmMutex wm; + struct _WsbmPMutex pm; + } *um = containerOf(m, union _PMutexConverter, wm); + + return &um->pm; +} + +static inline struct _WsbmPCond * +pCondConvert(struct _WsbmCond *c) +{ + union _PCondConverter + { + struct _WsbmCond wc; + struct _WsbmPCond pc; + } *uc = containerOf(c, union _PCondConverter, wc); + + return &uc->pc; +} + +static int +p_mutexInit(struct _WsbmMutex *mutex, struct _WsbmThreadFuncs *func) +{ + struct _WsbmPMutex *pMutex = pMutexConvert(mutex); + + if (sizeof(struct _WsbmMutex) < sizeof(struct _WsbmPMutex)) + return -EINVAL; + + pMutex->func = func; + pthread_mutex_init(&pMutex->mutex, NULL); + return 0; +} + +static void +p_mutexFree(struct _WsbmMutex *mutex) +{ + struct _WsbmPMutex *pMutex = pMutexConvert(mutex); + + pthread_mutex_destroy(&pMutex->mutex); +} + +static void +p_mutexLock(struct _WsbmMutex *mutex) +{ + struct _WsbmPMutex *pMutex = pMutexConvert(mutex); + + pthread_mutex_lock(&pMutex->mutex); +} + +static void +p_mutexUnlock(struct _WsbmMutex *mutex) +{ + struct _WsbmPMutex *pMutex = pMutexConvert(mutex); + + pthread_mutex_unlock(&pMutex->mutex); +} + +static int +p_condInit(struct _WsbmCond *cond, struct _WsbmThreadFuncs *func) +{ + struct _WsbmPCond *pCond = pCondConvert(cond); + + if (sizeof(struct _WsbmCond) < sizeof(struct _WsbmPCond)) + return -EINVAL; + + pCond->func = func; + pthread_cond_init(&pCond->cond, NULL); + return 0; +} + +static void +p_condFree(struct _WsbmCond *cond) +{ + struct _WsbmPCond *pCond = pCondConvert(cond); + + pthread_cond_destroy(&pCond->cond); +} + +static void +p_condBroadcast(struct _WsbmCond *cond) +{ + struct _WsbmPCond *pCond = pCondConvert(cond); + + pthread_cond_broadcast(&pCond->cond); +} + +static void +p_condWait(struct _WsbmCond *cond, struct _WsbmMutex *mutex) +{ + struct _WsbmPCond *pCond = pCondConvert(cond); + struct _WsbmPMutex *pMutex = pMutexConvert(mutex); + + pthread_cond_wait(&pCond->cond, &pMutex->mutex); +} + +static struct _WsbmThreadFuncs pthreadFunc = { + .mutexInit = p_mutexInit, + .mutexFree = p_mutexFree, + .mutexLock = p_mutexLock, + .mutexUnlock = p_mutexUnlock, + .condInit = p_condInit, + .condFree = p_condFree, + .condWait = p_condWait, + .condBroadcast = p_condBroadcast +}; + +struct _WsbmThreadFuncs * +wsbmPThreadFuncs(void) +{ + return &pthreadFunc; +} + +#else +#warning Pthreads is not present. Compiling without. + +struct _WsbmThreadFuncs * +wsbmPThreadFuncs(void) +{ + return &pthreadFunc; +} + +#endif diff --git a/src/wsbm_driver.h b/src/wsbm_driver.h new file mode 100644 index 0000000..d7da44e --- /dev/null +++ b/src/wsbm_driver.h @@ -0,0 +1,131 @@ +/************************************************************************** + * + * Copyright 2008 Tungsten Graphics, Inc., Cedar Park, TX., USA + * All Rights Reserved. + * Copyright 2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: + * Thomas Hellström + */ + +#ifndef _WSBM_DRIVER_H +#define _WSBM_DRIVER_H +#include +#include "wsbm_util.h" + +#define WSBM_MUTEX_SPACE 16 +#define WSBM_COND_SPACE 16 + +struct _WsbmMutex +{ + struct _WsbmThreadFuncs *func; + unsigned long storage[WSBM_MUTEX_SPACE]; +}; + +struct _WsbmCond +{ + struct _WsbmThreadFuncs *func; + unsigned long storage[WSBM_COND_SPACE]; +}; + +struct _WsbmThreadFuncs +{ + int (*mutexInit) (struct _WsbmMutex *, struct _WsbmThreadFuncs *); + void (*mutexFree) (struct _WsbmMutex *); + void (*mutexLock) (struct _WsbmMutex *); + void (*mutexUnlock) (struct _WsbmMutex *); + int (*condInit) (struct _WsbmCond *, struct _WsbmThreadFuncs *); + void (*condFree) (struct _WsbmCond *); + void (*condWait) (struct _WsbmCond *, struct _WsbmMutex *); + void (*condBroadcast) (struct _WsbmCond *); +}; + +extern struct _WsbmThreadFuncs *wsbmCurThreadFunc; + +#define WSBM_MUTEX_INIT(_mutex) \ + wsbmThreadFuncs()->mutexInit(_mutex,wsbmThreadFuncs()) +#define WSBM_MUTEX_FREE(_mutex) \ + { \ + (_mutex)->func->mutexFree(_mutex); \ + } +#define WSBM_MUTEX_LOCK(_mutex) \ + (_mutex)->func->mutexLock(_mutex); +#define WSBM_MUTEX_UNLOCK(_mutex) \ + (_mutex)->func->mutexUnlock(_mutex); + +#define WSBM_COND_INIT(_mutex) \ + wsbmThreadFuncs()->condInit(_mutex, wsbmThreadFuncs()) +#define WSBM_COND_FREE(_cond) \ + { \ + (_cond)->func->condFree(_cond); \ + } +#define WSBM_COND_WAIT(_cond, _mutex) \ + (_cond)->func->condWait(_cond, _mutex); +#define WSBM_COND_BROADCAST(_cond) \ + (_cond)->func->condBroadcast(_cond); + +struct _WsbmVNodeFuncs +{ + struct _ValidateNode *(*alloc) (struct _WsbmVNodeFuncs *, int); + void (*free) (struct _ValidateNode *); + void (*clear) (struct _ValidateNode *); +}; + +extern struct _WsbmVNodeFuncs *wsbmCurVNodeFunc; + +struct _WsbmBufStorage; +struct _WsbmKernelBuf; + +struct _ValidateNode +{ + uint32_t hash; + int type_id; + struct _WsbmListHead head; + struct _WsbmListHead hashHead; + int listItem; + uint64_t set_flags; + uint64_t clr_flags; + void *buf; + struct _WsbmVNodeFuncs *func; +}; + +static inline struct _WsbmVNodeFuncs * +wsbmVNodeFuncs(void) +{ + return wsbmCurVNodeFunc; +} + +static inline struct _WsbmThreadFuncs * +wsbmThreadFuncs(void) +{ + return wsbmCurThreadFunc; +} + +extern struct _WsbmThreadFuncs *wsbmNullThreadFuncs(void); + +extern struct _WsbmThreadFuncs *wsbmPThreadFuncs(void); + +#endif diff --git a/src/wsbm_fencemgr.c b/src/wsbm_fencemgr.c new file mode 100644 index 0000000..0e7a302 --- /dev/null +++ b/src/wsbm_fencemgr.c @@ -0,0 +1,474 @@ +/************************************************************************** + * + * Copyright 2008 Tungsten Graphics, Inc., Cedar Park, Tx., USA + * All Rights Reserved. + * Copyright 2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "wsbm_fencemgr.h" +#include "wsbm_pool.h" +#include "wsbm_manager.h" +#include +#include +#include +#include + +struct _WsbmFenceClass +{ + struct _WsbmListHead head; + struct _WsbmMutex mutex; + struct _WsbmMutex cmd_mutex; +}; + +/* + * Note: The struct _WsbmFenceMgr::Mutex should never be held + * during sleeps, since that may block fast concurrent access to + * fence data. + */ + +struct _WsbmFenceMgr +{ + /* + * Constant members. Need no mutex protection. + */ + struct _WsbmFenceMgrCreateInfo info; + void *private; + + /* + * Atomic members. No mutex protection. + */ + + struct _WsbmAtomic count; + + /* + * These members are protected by this->mutex + */ + + struct _WsbmFenceClass *classes; + uint32_t num_classes; +}; + +struct _WsbmFenceObject +{ + + /* + * These members are constant and need no mutex protection. + * Note that @private may point to a structure with its own + * mutex protection, that we don't care about. + */ + + struct _WsbmFenceMgr *mgr; + uint32_t fence_class; + uint32_t fence_type; + void *private; + + /* + * Atomic members. No mutex protection. note that + * @signaled types is updated using a compare-and-swap + * scheme to guarantee atomicity. + */ + + struct _WsbmAtomic refCount; + struct _WsbmAtomic signaled_types; + + /* + * These members are protected by mgr->mutex. + */ + struct _WsbmListHead head; +}; + +uint32_t +wsbmFenceType(struct _WsbmFenceObject *fence) +{ + return fence->fence_type; +} + +struct _WsbmFenceMgr * +wsbmFenceMgrCreate(const struct _WsbmFenceMgrCreateInfo *info) +{ + struct _WsbmFenceMgr *tmp; + uint32_t i, j; + int ret; + + tmp = calloc(1, sizeof(*tmp)); + if (!tmp) + return NULL; + + tmp->info = *info; + tmp->classes = calloc(tmp->info.num_classes, sizeof(*tmp->classes)); + if (!tmp->classes) + goto out_err; + + for (i = 0; i < tmp->info.num_classes; ++i) { + struct _WsbmFenceClass *fc = &tmp->classes[i]; + + WSBMINITLISTHEAD(&fc->head); + ret = WSBM_MUTEX_INIT(&fc->mutex); + if (ret) + goto out_err1; + ret = WSBM_MUTEX_INIT(&fc->cmd_mutex); + if (ret) { + WSBM_MUTEX_FREE(&fc->mutex); + goto out_err1; + } + } + wsbmAtomicSet(&tmp->count, 0); + + return tmp; + + out_err1: + for (j = 0; j < i; ++j) { + WSBM_MUTEX_FREE(&tmp->classes[j].mutex); + WSBM_MUTEX_FREE(&tmp->classes[j].cmd_mutex); + } + free(tmp->classes); + out_err: + if (tmp) + free(tmp); + return NULL; +} + +void +wsbmFenceUnreference(struct _WsbmFenceObject **pFence) +{ + struct _WsbmFenceObject *fence = *pFence; + struct _WsbmFenceMgr *mgr; + + *pFence = NULL; + if (fence == NULL) + return; + + mgr = fence->mgr; + if (wsbmAtomicDecZero(&fence->refCount)) { + struct _WsbmFenceClass *fc = &mgr->classes[fence->fence_class]; + + WSBM_MUTEX_LOCK(&fc->mutex); + WSBMLISTDELINIT(&fence->head); + WSBM_MUTEX_UNLOCK(&fc->mutex); + if (fence->private) + mgr->info.unreference(mgr, &fence->private); + fence->mgr = NULL; + wsbmAtomicDecZero(&mgr->count); + free(fence); + } +} + +static void +wsbmSignalPreviousFences(struct _WsbmFenceMgr *mgr, + struct _WsbmListHead *list, + uint32_t fence_class, uint32_t signaled_types) +{ + struct _WsbmFenceClass *fc = &mgr->classes[fence_class]; + struct _WsbmFenceObject *entry; + struct _WsbmListHead *prev; + uint32_t old_signaled_types; + uint32_t ret_st; + + WSBM_MUTEX_LOCK(&fc->mutex); + while (list != &fc->head && list->next != list) { + entry = WSBMLISTENTRY(list, struct _WsbmFenceObject, head); + + prev = list->prev; + + do { + old_signaled_types = wsbmAtomicRead(&entry->signaled_types); + signaled_types = + old_signaled_types | (signaled_types & entry->fence_type); + if (signaled_types == old_signaled_types) + break; + + ret_st = + wsbmAtomicCmpXchg(&entry->signaled_types, old_signaled_types, + signaled_types); + } while (ret_st != old_signaled_types); + + if (signaled_types == entry->fence_type) + WSBMLISTDELINIT(list); + + list = prev; + } + WSBM_MUTEX_UNLOCK(&fc->mutex); +} + +int +wsbmFenceFinish(struct _WsbmFenceObject *fence, uint32_t fence_type, + int lazy_hint) +{ + struct _WsbmFenceMgr *mgr = fence->mgr; + int ret = 0; + + if ((wsbmAtomicRead(&fence->signaled_types) & fence_type) == fence_type) + goto out; + + ret = mgr->info.finish(mgr, fence->private, fence_type, lazy_hint); + if (ret) + goto out; + + wsbmSignalPreviousFences(mgr, &fence->head, fence->fence_class, + fence_type); + out: + return ret; +} + +uint32_t +wsbmFenceSignaledTypeCached(struct _WsbmFenceObject * fence) +{ + return wsbmAtomicRead(&fence->signaled_types); +} + +int +wsbmFenceSignaledType(struct _WsbmFenceObject *fence, uint32_t flush_type, + uint32_t * signaled) +{ + int ret = 0; + struct _WsbmFenceMgr *mgr; + uint32_t signaled_types; + uint32_t old_signaled_types; + uint32_t ret_st; + + mgr = fence->mgr; + *signaled = wsbmAtomicRead(&fence->signaled_types); + if ((*signaled & flush_type) == flush_type) + goto out0; + + ret = mgr->info.signaled(mgr, fence->private, flush_type, signaled); + if (ret) { + *signaled = wsbmAtomicRead(&fence->signaled_types); + goto out0; + } + + do { + old_signaled_types = wsbmAtomicRead(&fence->signaled_types); + signaled_types = old_signaled_types | *signaled; + if (signaled_types == old_signaled_types) + break; + + ret_st = wsbmAtomicCmpXchg(&fence->signaled_types, old_signaled_types, + signaled_types); + if (old_signaled_types == ret_st) + wsbmSignalPreviousFences(mgr, &fence->head, fence->fence_class, + *signaled); + } while (old_signaled_types != ret_st); + + return 0; + out0: + return ret; +} + +struct _WsbmFenceObject * +wsbmFenceReference(struct _WsbmFenceObject *fence) +{ + if (fence == NULL) + return NULL; + wsbmAtomicInc(&fence->refCount); + return fence; +} + +struct _WsbmFenceObject * +wsbmFenceCreateSig(struct _WsbmFenceMgr *mgr, uint32_t fence_class, + uint32_t fence_type, uint32_t signaled_types, + void *private, size_t private_size) +{ + struct _WsbmFenceClass *fc = &mgr->classes[fence_class]; + struct _WsbmFenceObject *fence; + size_t fence_size = sizeof(*fence); + + if (private_size) + fence_size = ((fence_size + 15) & ~15); + + fence = calloc(1, fence_size + private_size); + + if (!fence) + goto out_err; + + wsbmAtomicSet(&fence->refCount, 1); + fence->mgr = mgr; + fence->fence_class = fence_class; + fence->fence_type = fence_type; + wsbmAtomicSet(&fence->signaled_types, signaled_types); + fence->private = private; + if (private_size) { + fence->private = (void *)(((uint8_t *) fence) + fence_size); + memcpy(fence->private, private, private_size); + } + + WSBM_MUTEX_LOCK(&fc->mutex); + WSBMLISTADDTAIL(&fence->head, &fc->head); + WSBM_MUTEX_UNLOCK(&fc->mutex); + wsbmAtomicInc(&mgr->count); + return fence; + + out_err: + { + int ret = mgr->info.finish(mgr, private, fence_type, 0); + + if (ret) + usleep(10000000); + } + if (fence) + free(fence); + + mgr->info.unreference(mgr, &private); + return NULL; +} + +struct _WsbmFenceObject * +wsbmFenceCreate(struct _WsbmFenceMgr *mgr, uint32_t fence_class, + uint32_t fence_type, void *private, size_t private_size) +{ + return wsbmFenceCreateSig(mgr, fence_class, fence_type, 0, private, + private_size); +} + +struct _WsbmTTMFenceMgrPriv +{ + int fd; + unsigned int devOffset; +}; + +static int +tSignaled(struct _WsbmFenceMgr *mgr, void *private, uint32_t flush_type, + uint32_t * signaled_type) +{ + struct _WsbmTTMFenceMgrPriv *priv = + (struct _WsbmTTMFenceMgrPriv *)mgr->private; + union ttm_fence_signaled_arg arg; + int ret; + + arg.req.handle = (unsigned long)private; + arg.req.fence_type = flush_type; + arg.req.flush = 1; + *signaled_type = 0; + + ret = drmCommandWriteRead(priv->fd, priv->devOffset + TTM_FENCE_SIGNALED, + &arg, sizeof(arg)); + if (ret) + return ret; + + *signaled_type = arg.rep.signaled_types; + return 0; +} + +static int +tFinish(struct _WsbmFenceMgr *mgr, void *private, uint32_t fence_type, + int lazy_hint) +{ + struct _WsbmTTMFenceMgrPriv *priv = + (struct _WsbmTTMFenceMgrPriv *)mgr->private; + union ttm_fence_finish_arg arg = + {.req = {.handle = (unsigned long)private, + .fence_type = fence_type, + .mode = (lazy_hint) ? TTM_FENCE_FINISH_MODE_LAZY : 0} + }; + int ret; + + do { + ret = drmCommandWriteRead(priv->fd, priv->devOffset + TTM_FENCE_FINISH, + &arg, sizeof(arg)); + } while (ret == -EAGAIN || ret == -ERESTART); + + return ret; +} + +static int +tUnref(struct _WsbmFenceMgr *mgr, void **private) +{ + struct _WsbmTTMFenceMgrPriv *priv = + (struct _WsbmTTMFenceMgrPriv *)mgr->private; + struct ttm_fence_unref_arg arg = {.handle = (unsigned long)*private }; + + *private = NULL; + + return drmCommandWrite(priv->fd, priv->devOffset + TTM_FENCE_UNREF, + &arg, sizeof(arg)); +} + +struct _WsbmFenceMgr * +wsbmFenceMgrTTMInit(int fd, unsigned int numClass, unsigned int devOffset) +{ + struct _WsbmFenceMgrCreateInfo info; + struct _WsbmFenceMgr *mgr; + struct _WsbmTTMFenceMgrPriv *priv = malloc(sizeof(*priv)); + + if (!priv) + return NULL; + + priv->fd = fd; + priv->devOffset = devOffset; + + info.flags = WSBM_FENCE_CLASS_ORDERED; + info.num_classes = numClass; + info.signaled = tSignaled; + info.finish = tFinish; + info.unreference = tUnref; + + mgr = wsbmFenceMgrCreate(&info); + if (mgr == NULL) { + free(priv); + return NULL; + } + + mgr->private = (void *)priv; + return mgr; +} + +void +wsbmFenceCmdLock(struct _WsbmFenceMgr *mgr, uint32_t fence_class) +{ + WSBM_MUTEX_LOCK(&mgr->classes[fence_class].cmd_mutex); +} + +void +wsbmFenceCmdUnlock(struct _WsbmFenceMgr *mgr, uint32_t fence_class) +{ + WSBM_MUTEX_UNLOCK(&mgr->classes[fence_class].cmd_mutex); +} + +void +wsbmFenceMgrTTMTakedown(struct _WsbmFenceMgr *mgr) +{ + int i; + + if (!mgr) + return; + + if (mgr->private) + free(mgr->private); + + for (i = 0; i < mgr->info.num_classes; ++i) { + WSBM_MUTEX_FREE(&mgr->classes[i].mutex); + WSBM_MUTEX_FREE(&mgr->classes[i].cmd_mutex); + } + free(mgr); + + return; +} diff --git a/src/wsbm_fencemgr.h b/src/wsbm_fencemgr.h new file mode 100644 index 0000000..cb10ae2 --- /dev/null +++ b/src/wsbm_fencemgr.h @@ -0,0 +1,162 @@ +/************************************************************************** + * + * Copyright 2008 Tungsten Graphics, Inc., Cedar Park, Tx., USA + * All Rights Reserved. + * Copyright 2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom + */ + +#ifndef WSBM_FENCEMGR_H +#define WSBM_FENCEMGR_H + +#include +#include + +struct _WsbmFenceObject; +struct _WsbmFenceMgr; + +/* + * Do a quick check to see if the fence manager has registered the fence + * object as signaled. Note that this function may return a false negative + * answer. + */ +extern uint32_t wsbmFenceSignaledTypeCached(struct _WsbmFenceObject *fence); + +/* + * Check if the fence object is signaled. This function can be substantially + * more expensive to call than the above function, but will not return a false + * negative answer. The argument "flush_type" sets the types that the + * underlying mechanism must make sure will eventually signal. + */ +extern int wsbmFenceSignaledType(struct _WsbmFenceObject *fence, + uint32_t flush_type, uint32_t * signaled); + +/* + * Convenience functions. + */ + +static inline int +wsbmFenceSignaled(struct _WsbmFenceObject *fence, uint32_t flush_type) +{ + uint32_t signaled_types; + int ret = wsbmFenceSignaledType(fence, flush_type, &signaled_types); + + if (ret) + return 0; + return ((signaled_types & flush_type) == flush_type); +} + +static inline int +wsbmFenceSignaledCached(struct _WsbmFenceObject *fence, uint32_t flush_type) +{ + uint32_t signaled_types = wsbmFenceSignaledTypeCached(fence); + + return ((signaled_types & flush_type) == flush_type); +} + +/* + * Reference a fence object. + */ +extern struct _WsbmFenceObject *wsbmFenceReference(struct _WsbmFenceObject + *fence); + +/* + * Unreference a fence object. The fence object pointer will be reset to NULL. + */ + +extern void wsbmFenceUnreference(struct _WsbmFenceObject **pFence); + +/* + * Wait for a fence to signal the indicated fence_type. + * If "lazy_hint" is true, it indicates that the wait may sleep to avoid + * busy-wait polling. + */ +extern int wsbmFenceFinish(struct _WsbmFenceObject *fence, + uint32_t fence_type, int lazy_hint); + +/* + * Create a WsbmFenceObject for manager "mgr". + * + * "private" is a pointer that should be used for the callbacks in + * struct _WsbmFenceMgrCreateInfo. + * + * if private_size is nonzero, then the info stored at *private, with size + * private size will be copied and the fence manager will instead use a + * pointer to the copied data for the callbacks in + * struct _WsbmFenceMgrCreateInfo. In that case, the object pointed to by + * "private" may be destroyed after the call to wsbmFenceCreate. + */ +extern struct _WsbmFenceObject *wsbmFenceCreate(struct _WsbmFenceMgr *mgr, + uint32_t fence_class, + uint32_t fence_type, + void *private, + size_t private_size); + + +extern struct _WsbmFenceObject *wsbmFenceCreateSig(struct _WsbmFenceMgr *mgr, + uint32_t fence_class, + uint32_t fence_type, + uint32_t signaled_types, + void *private, + size_t private_size); + +extern uint32_t wsbmFenceType(struct _WsbmFenceObject *fence); + +/* + * Fence creations are ordered. If a fence signals a fence_type, + * it is safe to assume that all fences of the same class that was + * created before that fence has signaled the same type. + */ + +#define WSBM_FENCE_CLASS_ORDERED (1 << 0) + +struct _WsbmFenceMgrCreateInfo +{ + uint32_t flags; + uint32_t num_classes; + int (*signaled) (struct _WsbmFenceMgr * mgr, void *private, + uint32_t flush_type, uint32_t * signaled_type); + int (*finish) (struct _WsbmFenceMgr * mgr, void *private, + uint32_t fence_type, int lazy_hint); + int (*unreference) (struct _WsbmFenceMgr * mgr, void **private); +}; + +extern struct _WsbmFenceMgr *wsbmFenceMgrCreate(const struct + _WsbmFenceMgrCreateInfo + *info); +extern void wsbmFenceCmdLock(struct _WsbmFenceMgr *mgr, uint32_t fence_class); +extern void wsbmFenceCmdUnlock(struct _WsbmFenceMgr *mgr, + uint32_t fence_class); +/* + * Builtin drivers. + */ + +extern struct _WsbmFenceMgr *wsbmFenceMgrTTMInit(int fd, + unsigned int numClass, + unsigned int devOffset); +extern void wsbmFenceMgrTTMTakedown(struct _WsbmFenceMgr *mgr); +#endif diff --git a/src/wsbm_mallocpool.c b/src/wsbm_mallocpool.c new file mode 100644 index 0000000..b5252d6 --- /dev/null +++ b/src/wsbm_mallocpool.c @@ -0,0 +1,187 @@ +/************************************************************************** + * + * Copyright 2008 Tungsten Graphics, Inc., Cedar Park, TX., USA + * All Rights Reserved. + * Copyright 2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellström + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include "wsbm_pool.h" +#include "wsbm_manager.h" + +struct _WsbmMallocBuffer +{ + struct _WsbmBufStorage buf; + size_t size; + void *mem; +}; + +static inline struct _WsbmMallocBuffer * +mallocBuf(struct _WsbmBufStorage *buf) +{ + return containerOf(buf, struct _WsbmMallocBuffer, buf); +} + +static struct _WsbmBufStorage * +pool_create(struct _WsbmBufferPool *pool, + unsigned long size, uint32_t placement, unsigned alignment) +{ + struct _WsbmMallocBuffer *mBuf = malloc(size + sizeof(*mBuf) + 16); + + if (!mBuf) + return NULL; + + wsbmBufStorageInit(&mBuf->buf, pool); + mBuf->size = size; + mBuf->mem = (void *)((unsigned long)mBuf + sizeof(*mBuf)); + if ((placement & WSBM_PL_MASK_MEM) != WSBM_PL_FLAG_SYSTEM) + abort(); + + return &mBuf->buf; +} + +static void +pool_destroy(struct _WsbmBufStorage **buf) +{ + free(mallocBuf(*buf)); + *buf = NULL; +} + +static int +pool_waitIdle(struct _WsbmBufStorage *buf, int lazy) +{ + return 0; +} + +static int +pool_map(struct _WsbmBufStorage *buf, unsigned mode, void **virtual) +{ + *virtual = mallocBuf(buf)->mem; + return 0; +} + +static void +pool_unmap(struct _WsbmBufStorage *buf) +{ + ; +} + +static int +pool_syncforcpu(struct _WsbmBufStorage *buf, unsigned mode) +{ + return 0; +} + +static void +pool_releasefromcpu(struct _WsbmBufStorage *buf, unsigned mode) +{ + ; +} + +static unsigned long +pool_offset(struct _WsbmBufStorage *buf) +{ + /* + * BUG + */ + abort(); + return 0UL; +} + +static unsigned long +pool_poolOffset(struct _WsbmBufStorage *buf) +{ + /* + * BUG + */ + abort(); +} + +static uint32_t +pool_placement(struct _WsbmBufStorage *buf) +{ + return WSBM_PL_FLAG_SYSTEM | WSBM_PL_FLAG_CACHED; +} + +static unsigned long +pool_size(struct _WsbmBufStorage *buf) +{ + return mallocBuf(buf)->size; +} + +static void +pool_fence(struct _WsbmBufStorage *buf, struct _WsbmFenceObject *fence) +{ + abort(); +} + +static struct _WsbmKernelBuf * +pool_kernel(struct _WsbmBufStorage *buf) +{ + abort(); + return NULL; +} + +static void +pool_takedown(struct _WsbmBufferPool *pool) +{ + free(pool); +} + +struct _WsbmBufferPool * +wsbmMallocPoolInit(void) +{ + struct _WsbmBufferPool *pool; + + pool = (struct _WsbmBufferPool *)calloc(1, sizeof(*pool)); + if (!pool) + return NULL; + + pool->fd = -1; + pool->map = &pool_map; + pool->unmap = &pool_unmap; + pool->syncforcpu = &pool_syncforcpu; + pool->releasefromcpu = &pool_releasefromcpu; + pool->destroy = &pool_destroy; + pool->offset = &pool_offset; + pool->poolOffset = &pool_poolOffset; + pool->placement = &pool_placement; + pool->size = &pool_size; + pool->create = &pool_create; + pool->fence = &pool_fence; + pool->kernel = &pool_kernel; + pool->validate = NULL; + pool->waitIdle = &pool_waitIdle; + pool->takeDown = &pool_takedown; + return pool; +} diff --git a/src/wsbm_manager.c b/src/wsbm_manager.c new file mode 100644 index 0000000..1c347c7 --- /dev/null +++ b/src/wsbm_manager.c @@ -0,0 +1,1225 @@ +/************************************************************************** + * + * Copyright 2006-2008 Tungsten Graphics, Inc., Cedar Park, TX., USA + * All Rights Reserved. + * Copyright 2009 Vmware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellström + * Keith Whitwell + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include "errno.h" +#include "string.h" +#include "wsbm_pool.h" +#include "wsbm_manager.h" +#include "wsbm_fencemgr.h" +#include "wsbm_driver.h" +#include "wsbm_priv.h" +#include "wsbm_util.h" +#include "wsbm_atomic.h" +#include "assert.h" + +#define WSBM_BODATA_SIZE_ACCEPT 4096 + +#define WSBM_BUFFER_COMPLEX 0 +#define WSBM_BUFFER_SIMPLE 1 +#define WSBM_BUFFER_REF 2 + +struct _ValidateList +{ + unsigned numTarget; + unsigned numCurrent; + unsigned numOnList; + unsigned hashSize; + uint32_t hashMask; + int driverData; + struct _WsbmListHead list; + struct _WsbmListHead free; + struct _WsbmListHead *hashTable; +}; + +struct _WsbmBufferObject +{ + /* Left to the client to protect this data for now. */ + + struct _WsbmAtomic refCount; + struct _WsbmBufStorage *storage; + + uint32_t placement; + unsigned alignment; + unsigned bufferType; + struct _WsbmBufferPool *pool; +}; + +struct _WsbmBufferList +{ + int hasKernelBuffers; + + struct _ValidateList kernelBuffers; /* List of kernel buffers needing validation */ + struct _ValidateList userBuffers; /* List of user-space buffers needing validation */ +}; + +static struct _WsbmMutex bmMutex; +static struct _WsbmCond bmCond; +static int initialized = 0; +static void *commonData = NULL; + +static int kernelReaders = 0; +static int kernelLocked = 0; + +int +wsbmInit(struct _WsbmThreadFuncs *tf, struct _WsbmVNodeFuncs *vf) +{ + int ret; + + wsbmCurThreadFunc = tf; + wsbmCurVNodeFunc = vf; + + ret = WSBM_MUTEX_INIT(&bmMutex); + if (ret) + return -ENOMEM; + ret = WSBM_COND_INIT(&bmCond); + if (ret) { + WSBM_MUTEX_FREE(&bmMutex); + return -ENOMEM; + } + + initialized = 1; + return 0; +} + +void +wsbmCommonDataSet(void *d) +{ + commonData = d; +} + +void * +wsbmCommonDataGet(void) +{ + return commonData; +} + +int +wsbmIsInitialized(void) +{ + return initialized; +} + +void +wsbmTakedown(void) +{ + initialized = 0; + commonData = NULL; + WSBM_COND_FREE(&bmCond); + WSBM_MUTEX_FREE(&bmMutex); +} + +static struct _ValidateNode * +validateListAddNode(struct _ValidateList *list, void *item, + uint32_t hash, uint64_t flags, uint64_t mask) +{ + struct _ValidateNode *node; + struct _WsbmListHead *l; + struct _WsbmListHead *hashHead; + + l = list->free.next; + if (l == &list->free) { + node = wsbmVNodeFuncs()->alloc(wsbmVNodeFuncs(), 0); + if (!node) { + return NULL; + } + list->numCurrent++; + } else { + WSBMLISTDEL(l); + node = WSBMLISTENTRY(l, struct _ValidateNode, head); + } + node->buf = item; + node->set_flags = flags & mask; + node->clr_flags = (~flags) & mask; + node->listItem = list->numOnList; + WSBMLISTADDTAIL(&node->head, &list->list); + list->numOnList++; + hashHead = list->hashTable + hash; + WSBMLISTADDTAIL(&node->hashHead, hashHead); + + return node; +} + +static uint32_t +wsbmHashFunc(uint8_t * key, uint32_t len, uint32_t mask) +{ + uint32_t hash, i; + + for (hash = 0, i = 0; i < len; ++i) { + hash += *key++; + hash += (hash << 10); + hash ^= (hash >> 6); + } + + hash += (hash << 3); + hash ^= (hash >> 11); + hash += (hash << 15); + + return hash & mask; +} + +static void +validateFreeList(struct _ValidateList *list) +{ + struct _ValidateNode *node; + struct _WsbmListHead *l; + + l = list->list.next; + while (l != &list->list) { + WSBMLISTDEL(l); + node = WSBMLISTENTRY(l, struct _ValidateNode, head); + + WSBMLISTDEL(&node->hashHead); + node->func->free(node); + l = list->list.next; + list->numCurrent--; + list->numOnList--; + } + + l = list->free.next; + while (l != &list->free) { + WSBMLISTDEL(l); + node = WSBMLISTENTRY(l, struct _ValidateNode, head); + + node->func->free(node); + l = list->free.next; + list->numCurrent--; + } + free(list->hashTable); +} + +static int +validateListAdjustNodes(struct _ValidateList *list) +{ + struct _ValidateNode *node; + struct _WsbmListHead *l; + int ret = 0; + + while (list->numCurrent < list->numTarget) { + node = wsbmVNodeFuncs()->alloc(wsbmVNodeFuncs(), list->driverData); + if (!node) { + ret = -ENOMEM; + break; + } + list->numCurrent++; + WSBMLISTADD(&node->head, &list->free); + } + + while (list->numCurrent > list->numTarget) { + l = list->free.next; + if (l == &list->free) + break; + WSBMLISTDEL(l); + node = WSBMLISTENTRY(l, struct _ValidateNode, head); + + node->func->free(node); + list->numCurrent--; + } + return ret; +} + +static inline int +wsbmPot(unsigned int val) +{ + unsigned int shift = 0; + while(val > (1 << shift)) + shift++; + + return shift; +} + + + +static int +validateCreateList(int numTarget, struct _ValidateList *list, int driverData) +{ + int i; + unsigned int shift = wsbmPot(numTarget); + int ret; + + list->hashSize = (1 << shift); + list->hashMask = list->hashSize - 1; + + list->hashTable = malloc(list->hashSize * sizeof(*list->hashTable)); + if (!list->hashTable) + return -ENOMEM; + + for (i = 0; i < list->hashSize; ++i) + WSBMINITLISTHEAD(&list->hashTable[i]); + + WSBMINITLISTHEAD(&list->list); + WSBMINITLISTHEAD(&list->free); + list->numTarget = numTarget; + list->numCurrent = 0; + list->numOnList = 0; + list->driverData = driverData; + ret = validateListAdjustNodes(list); + if (ret != 0) + free(list->hashTable); + + return ret; +} + +static int +validateResetList(struct _ValidateList *list) +{ + struct _WsbmListHead *l; + struct _ValidateNode *node; + int ret; + + ret = validateListAdjustNodes(list); + if (ret) + return ret; + + l = list->list.next; + while (l != &list->list) { + WSBMLISTDEL(l); + node = WSBMLISTENTRY(l, struct _ValidateNode, head); + + WSBMLISTDEL(&node->hashHead); + WSBMLISTADD(l, &list->free); + list->numOnList--; + l = list->list.next; + } + return validateListAdjustNodes(list); +} + +void +wsbmWriteLockKernelBO(void) +{ + WSBM_MUTEX_LOCK(&bmMutex); + while (kernelReaders != 0) + WSBM_COND_WAIT(&bmCond, &bmMutex); + kernelLocked = 1; +} + +void +wsbmWriteUnlockKernelBO(void) +{ + kernelLocked = 0; + WSBM_MUTEX_UNLOCK(&bmMutex); +} + +void +wsbmReadLockKernelBO(void) +{ + WSBM_MUTEX_LOCK(&bmMutex); + if (kernelReaders++ == 0) + kernelLocked = 1; + WSBM_MUTEX_UNLOCK(&bmMutex); +} + +void +wsbmReadUnlockKernelBO(void) +{ + WSBM_MUTEX_LOCK(&bmMutex); + if (--kernelReaders == 0) { + kernelLocked = 0; + WSBM_COND_BROADCAST(&bmCond); + } + WSBM_MUTEX_UNLOCK(&bmMutex); +} + +void +wsbmBOWaitIdle(struct _WsbmBufferObject *buf, int lazy) +{ + struct _WsbmBufStorage *storage; + + storage = buf->storage; + if (!storage) + return; + + (void)storage->pool->waitIdle(storage, lazy); +} + +void * +wsbmBOMap(struct _WsbmBufferObject *buf, unsigned mode) +{ + struct _WsbmBufStorage *storage = buf->storage; + void *virtual; + int retval; + + retval = storage->pool->map(storage, mode, &virtual); + + return (retval == 0) ? virtual : NULL; +} + +void +wsbmBOUnmap(struct _WsbmBufferObject *buf) +{ + struct _WsbmBufStorage *storage = buf->storage; + + if (!storage) + return; + + storage->pool->unmap(storage); +} + +int +wsbmBOSyncForCpu(struct _WsbmBufferObject *buf, unsigned mode) +{ + struct _WsbmBufStorage *storage = buf->storage; + + return storage->pool->syncforcpu(storage, mode); +} + +void +wsbmBOReleaseFromCpu(struct _WsbmBufferObject *buf, unsigned mode) +{ + struct _WsbmBufStorage *storage = buf->storage; + + storage->pool->releasefromcpu(storage, mode); +} + +unsigned long +wsbmBOOffsetHint(struct _WsbmBufferObject *buf) +{ + struct _WsbmBufStorage *storage = buf->storage; + + return storage->pool->offset(storage); +} + +unsigned long +wsbmBOPoolOffset(struct _WsbmBufferObject *buf) +{ + struct _WsbmBufStorage *storage = buf->storage; + + return storage->pool->poolOffset(storage); +} + +uint32_t +wsbmBOPlacementHint(struct _WsbmBufferObject * buf) +{ + struct _WsbmBufStorage *storage = buf->storage; + + assert(buf->storage != NULL); + + return storage->pool->placement(storage); +} + +struct _WsbmBufferObject * +wsbmBOReference(struct _WsbmBufferObject *buf) +{ + if (buf->bufferType == WSBM_BUFFER_SIMPLE) { + wsbmAtomicInc(&buf->storage->refCount); + } else { + wsbmAtomicInc(&buf->refCount); + } + return buf; +} + +int +wsbmBOSetStatus(struct _WsbmBufferObject *buf, + uint32_t setFlags, uint32_t clrFlags) +{ + struct _WsbmBufStorage *storage = buf->storage; + + if (!storage) + return 0; + + if (storage->pool->setStatus == NULL) + return -EINVAL; + + return storage->pool->setStatus(storage, setFlags, clrFlags); +} + +void +wsbmBOUnreference(struct _WsbmBufferObject **p_buf) +{ + struct _WsbmBufferObject *buf = *p_buf; + + *p_buf = NULL; + + if (!buf) + return; + + if (buf->bufferType == WSBM_BUFFER_SIMPLE) { + struct _WsbmBufStorage *dummy = buf->storage; + + wsbmBufStorageUnref(&dummy); + return; + } + + if (wsbmAtomicDecZero(&buf->refCount)) { + wsbmBufStorageUnref(&buf->storage); + free(buf); + } +} + +int +wsbmBOData(struct _WsbmBufferObject *buf, + unsigned size, const void *data, + struct _WsbmBufferPool *newPool, uint32_t placement) +{ + void *virtual = NULL; + int newBuffer; + int retval = 0; + struct _WsbmBufStorage *storage; + int synced = 0; + uint32_t placement_diff; + struct _WsbmBufferPool *curPool; + + if (buf->bufferType == WSBM_BUFFER_SIMPLE) + return -EINVAL; + + storage = buf->storage; + + if (newPool == NULL) + newPool = buf->pool; + + if (newPool == NULL) + return -EINVAL; + + newBuffer = (!storage || storage->pool != newPool || + storage->pool->size(storage) < size || + storage->pool->size(storage) > + size + WSBM_BODATA_SIZE_ACCEPT); + + if (!placement) + placement = buf->placement; + + if (newBuffer) { + if (buf->bufferType == WSBM_BUFFER_REF) + return -EINVAL; + + wsbmBufStorageUnref(&buf->storage); + + if (size == 0) { + buf->pool = newPool; + buf->placement = placement; + retval = 0; + goto out; + } + + buf->storage = + newPool->create(newPool, size, placement, buf->alignment); + if (!buf->storage) { + retval = -ENOMEM; + goto out; + } + + buf->placement = placement; + buf->pool = newPool; + } else if (wsbmAtomicRead(&storage->onList) || + 0 != storage->pool->syncforcpu(storage, WSBM_SYNCCPU_WRITE | + WSBM_SYNCCPU_DONT_BLOCK)) { + /* + * Buffer is busy. need to create a new one. + */ + + struct _WsbmBufStorage *tmp_storage; + + curPool = storage->pool; + + tmp_storage = + curPool->create(curPool, size, placement, buf->alignment); + + if (tmp_storage) { + wsbmBufStorageUnref(&buf->storage); + buf->storage = tmp_storage; + buf->placement = placement; + } else { + retval = curPool->syncforcpu(storage, WSBM_SYNCCPU_WRITE); + if (retval) + goto out; + synced = 1; + } + } else + synced = 1; + + placement_diff = placement ^ buf->placement; + + /* + * We might need to change buffer placement. + */ + + storage = buf->storage; + curPool = storage->pool; + + if (placement_diff) { + assert(curPool->setStatus != NULL); + curPool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE); + retval = curPool->setStatus(storage, + placement_diff & placement, + placement_diff & ~placement); + if (retval) + goto out; + + buf->placement = placement; + + } + + if (!synced) { + retval = curPool->syncforcpu(buf->storage, WSBM_SYNCCPU_WRITE); + + if (retval) + goto out; + synced = 1; + } + + storage = buf->storage; + curPool = storage->pool; + + if (data) { + retval = curPool->map(storage, WSBM_ACCESS_WRITE, &virtual); + if (retval) + goto out; + memcpy(virtual, data, size); + curPool->unmap(storage); + } + + out: + + if (synced) + curPool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE); + + return retval; +} + +static struct _WsbmBufStorage * +wsbmStorageClone(struct _WsbmBufferObject *buf) +{ + struct _WsbmBufStorage *storage = buf->storage; + struct _WsbmBufferPool *pool = storage->pool; + + return pool->create(pool, pool->size(storage), buf->placement, + buf->alignment); +} + +struct _WsbmBufferObject * +wsbmBOClone(struct _WsbmBufferObject *buf, + int (*accelCopy) (struct _WsbmBufferObject *, + struct _WsbmBufferObject *)) +{ + struct _WsbmBufferObject *newBuf; + int ret; + + newBuf = malloc(sizeof(*newBuf)); + if (!newBuf) + return NULL; + + *newBuf = *buf; + newBuf->storage = wsbmStorageClone(buf); + if (!newBuf->storage) + goto out_err0; + + wsbmAtomicSet(&newBuf->refCount, 1); + if (!accelCopy || accelCopy(newBuf, buf) != 0) { + + struct _WsbmBufferPool *pool = buf->storage->pool; + struct _WsbmBufStorage *storage = buf->storage; + struct _WsbmBufStorage *newStorage = newBuf->storage; + void *virtual; + void *nVirtual; + + ret = pool->syncforcpu(storage, WSBM_SYNCCPU_READ); + if (ret) + goto out_err1; + ret = pool->map(storage, WSBM_ACCESS_READ, &virtual); + if (ret) + goto out_err2; + ret = pool->map(newStorage, WSBM_ACCESS_WRITE, &nVirtual); + if (ret) + goto out_err3; + + memcpy(nVirtual, virtual, pool->size(storage)); + pool->unmap(newBuf->storage); + pool->unmap(buf->storage); + pool->releasefromcpu(storage, WSBM_SYNCCPU_READ); + } + + return newBuf; + out_err3: + buf->pool->unmap(buf->storage); + out_err2: + buf->pool->releasefromcpu(buf->storage, WSBM_SYNCCPU_READ); + out_err1: + wsbmBufStorageUnref(&newBuf->storage); + out_err0: + free(newBuf); + return 0; +} + +int +wsbmBOSubData(struct _WsbmBufferObject *buf, + unsigned long offset, unsigned long size, const void *data, + int (*accelCopy) (struct _WsbmBufferObject *, + struct _WsbmBufferObject *)) +{ + int ret = 0; + + if (buf->bufferType == WSBM_BUFFER_SIMPLE) + return -EINVAL; + + if (size && data) { + void *virtual; + struct _WsbmBufStorage *storage = buf->storage; + struct _WsbmBufferPool *pool = storage->pool; + + ret = pool->syncforcpu(storage, WSBM_SYNCCPU_WRITE); + if (ret) + goto out; + + if (wsbmAtomicRead(&storage->onList)) { + + struct _WsbmBufferObject *newBuf; + + /* + * Another context has this buffer on its validate list. + * This should be a very rare situation, but it can be valid, + * and therefore we must deal with it by cloning the storage. + */ + + pool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE); + newBuf = wsbmBOClone(buf, accelCopy); + + /* + * If clone fails we have the choice of either bailing. + * (The other context will be happy), or go on and update + * the old buffer anyway. (We will be happy). We choose the + * latter. + */ + + if (newBuf) { + storage = newBuf->storage; + wsbmAtomicInc(&storage->refCount); + wsbmBufStorageUnref(&buf->storage); + buf->storage = storage; + wsbmBOUnreference(&newBuf); + pool = storage->pool; + } + + ret = pool->syncforcpu(storage, WSBM_SYNCCPU_WRITE); + if (ret) + goto out; + } + + ret = pool->map(storage, WSBM_ACCESS_WRITE, &virtual); + if (ret) { + pool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE); + goto out; + } + + memcpy((unsigned char *)virtual + offset, data, size); + pool->unmap(storage); + pool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE); + } + out: + return ret; +} + +int +wsbmBOGetSubData(struct _WsbmBufferObject *buf, + unsigned long offset, unsigned long size, void *data) +{ + int ret = 0; + + if (size && data) { + void *virtual; + struct _WsbmBufStorage *storage = buf->storage; + struct _WsbmBufferPool *pool = storage->pool; + + ret = pool->syncforcpu(storage, WSBM_SYNCCPU_READ); + if (ret) + goto out; + ret = pool->map(storage, WSBM_ACCESS_READ, &virtual); + if (ret) { + pool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE); + goto out; + } + memcpy(data, (unsigned char *)virtual + offset, size); + pool->unmap(storage); + pool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE); + } + out: + return ret; +} + +int +wsbmBOSetReferenced(struct _WsbmBufferObject *buf, unsigned long handle) +{ + int ret = 0; + + wsbmBufStorageUnref(&buf->storage); + if (buf->pool->createByReference == NULL) { + ret = -EINVAL; + goto out; + } + buf->storage = buf->pool->createByReference(buf->pool, handle); + if (!buf->storage) { + ret = -EINVAL; + goto out; + } + buf->bufferType = WSBM_BUFFER_REF; + out: + return ret; +} + +void +wsbmBOFreeSimple(void *ptr) +{ + free(ptr); +} + +struct _WsbmBufferObject * +wsbmBOCreateSimple(struct _WsbmBufferPool *pool, + unsigned long size, + uint32_t placement, + unsigned alignment, size_t extra_size, size_t * offset) +{ + struct _WsbmBufferObject *buf; + struct _WsbmBufStorage *storage; + + *offset = (sizeof(*buf) + 15) & ~15; + + if (extra_size) { + extra_size += *offset - sizeof(*buf); + } + + buf = (struct _WsbmBufferObject *)calloc(1, sizeof(*buf) + extra_size); + if (!buf) + return NULL; + + storage = pool->create(pool, size, placement, alignment); + if (!storage) + goto out_err0; + + storage->destroyContainer = &wsbmBOFreeSimple; + storage->destroyArg = buf; + + buf->storage = storage; + buf->alignment = alignment; + buf->pool = pool; + buf->placement = placement; + buf->bufferType = WSBM_BUFFER_SIMPLE; + + return buf; + + out_err0: + free(buf); + return NULL; +} + +int +wsbmGenBuffers(struct _WsbmBufferPool *pool, + unsigned n, + struct _WsbmBufferObject *buffers[], + unsigned alignment, uint32_t placement) +{ + struct _WsbmBufferObject *buf; + int i; + + placement = (placement) ? placement : + WSBM_PL_FLAG_SYSTEM | WSBM_PL_FLAG_CACHED; + + for (i = 0; i < n; ++i) { + buf = (struct _WsbmBufferObject *)calloc(1, sizeof(*buf)); + if (!buf) + return -ENOMEM; + + wsbmAtomicSet(&buf->refCount, 1); + buf->placement = placement; + buf->alignment = alignment; + buf->pool = pool; + buf->bufferType = WSBM_BUFFER_COMPLEX; + buffers[i] = buf; + } + return 0; +} + +void +wsbmDeleteBuffers(unsigned n, struct _WsbmBufferObject *buffers[]) +{ + int i; + + for (i = 0; i < n; ++i) { + wsbmBOUnreference(&buffers[i]); + } +} + +/* + * Note that lists are per-context and don't need mutex protection. + */ + +struct _WsbmBufferList * +wsbmBOCreateList(int target, int hasKernelBuffers) +{ + struct _WsbmBufferList *list = calloc(sizeof(*list), 1); + int ret; + + list->hasKernelBuffers = hasKernelBuffers; + if (hasKernelBuffers) { + ret = validateCreateList(target, &list->kernelBuffers, 0); + if (ret) + return NULL; + } + + ret = validateCreateList(target, &list->userBuffers, 1); + if (ret) { + validateFreeList(&list->kernelBuffers); + return NULL; + } + + return list; +} + +int +wsbmBOResetList(struct _WsbmBufferList *list) +{ + int ret; + + if (list->hasKernelBuffers) { + ret = validateResetList(&list->kernelBuffers); + if (ret) + return ret; + } + ret = validateResetList(&list->userBuffers); + return ret; +} + +void +wsbmBOFreeList(struct _WsbmBufferList *list) +{ + if (list->hasKernelBuffers) + validateFreeList(&list->kernelBuffers); + validateFreeList(&list->userBuffers); + free(list); +} + +static int +wsbmAddValidateItem(struct _ValidateList *list, void *buf, uint64_t flags, + uint64_t mask, int *itemLoc, + struct _ValidateNode **pnode, int *newItem) +{ + struct _ValidateNode *node, *cur; + struct _WsbmListHead *l; + struct _WsbmListHead *hashHead; + uint32_t hash; + uint32_t count = 0; + uint32_t key = (unsigned long) buf; + + cur = NULL; + hash = wsbmHashFunc((uint8_t *) &key, 4, list->hashMask); + hashHead = list->hashTable + hash; + *newItem = 0; + + for (l = hashHead->next; l != hashHead; l = l->next) { + count++; + node = WSBMLISTENTRY(l, struct _ValidateNode, hashHead); + + if (node->buf == buf) { + cur = node; + break; + } + } + + if (!cur) { + cur = validateListAddNode(list, buf, hash, flags, mask); + if (!cur) + return -ENOMEM; + *newItem = 1; + cur->func->clear(cur); + } else { + uint64_t set_flags = flags & mask; + uint64_t clr_flags = (~flags) & mask; + + if (((cur->clr_flags | clr_flags) & WSBM_PL_MASK_MEM) == + WSBM_PL_MASK_MEM) { + /* + * No available memory type left. Bail. + */ + return -EINVAL; + } + + if ((cur->set_flags | set_flags) & + (cur->clr_flags | clr_flags) & ~WSBM_PL_MASK_MEM) { + /* + * Conflicting flags. Bail. + */ + return -EINVAL; + } + + cur->set_flags &= ~(clr_flags & WSBM_PL_MASK_MEM); + cur->set_flags |= (set_flags & ~WSBM_PL_MASK_MEM); + cur->clr_flags |= clr_flags; + } + *itemLoc = cur->listItem; + if (pnode) + *pnode = cur; + return 0; +} + +int +wsbmBOAddListItem(struct _WsbmBufferList *list, + struct _WsbmBufferObject *buf, + uint64_t flags, uint64_t mask, int *itemLoc, + struct _ValidateNode **node) +{ + int newItem; + struct _WsbmBufStorage *storage = buf->storage; + int ret; + int dummy; + struct _ValidateNode *dummyNode; + + if (list->hasKernelBuffers) { + ret = wsbmAddValidateItem(&list->kernelBuffers, + storage->pool->kernel(storage), + flags, mask, itemLoc, node, &dummy); + if (ret) + goto out_unlock; + } else { + *node = NULL; + *itemLoc = -1000; + } + + ret = wsbmAddValidateItem(&list->userBuffers, storage, + flags, mask, &dummy, &dummyNode, &newItem); + if (ret) + goto out_unlock; + + if (newItem) { + wsbmAtomicInc(&storage->refCount); + wsbmAtomicInc(&storage->onList); + } + + out_unlock: + return ret; +} + +void +wsbmBOFence(struct _WsbmBufferObject *buf, struct _WsbmFenceObject *fence) +{ + struct _WsbmBufStorage *storage; + + storage = buf->storage; + if (storage->pool->fence) + storage->pool->fence(storage, fence); + +} + +int +wsbmBOOnList(const struct _WsbmBufferObject *buf) +{ + if (buf->storage == NULL) + return 0; + return wsbmAtomicRead(&buf->storage->onList); +} + +int +wsbmBOUnrefUserList(struct _WsbmBufferList *list) +{ + struct _WsbmBufStorage *storage; + void *curBuf; + + curBuf = validateListIterator(&list->userBuffers); + + while (curBuf) { + storage = (struct _WsbmBufStorage *)(validateListNode(curBuf)->buf); + wsbmAtomicDec(&storage->onList); + wsbmBufStorageUnref(&storage); + curBuf = validateListNext(&list->userBuffers, curBuf); + } + + return wsbmBOResetList(list); +} + + +int +wsbmBOFenceUserList(struct _WsbmBufferList *list, + struct _WsbmFenceObject *fence) +{ + struct _WsbmBufStorage *storage; + void *curBuf; + + curBuf = validateListIterator(&list->userBuffers); + + /* + * User-space fencing callbacks. + */ + + while (curBuf) { + storage = (struct _WsbmBufStorage *)(validateListNode(curBuf)->buf); + + storage->pool->fence(storage, fence); + wsbmAtomicDec(&storage->onList); + wsbmBufStorageUnref(&storage); + curBuf = validateListNext(&list->userBuffers, curBuf); + } + + return wsbmBOResetList(list); +} + +int +wsbmBOValidateUserList(struct _WsbmBufferList *list) +{ + void *curBuf; + struct _WsbmBufStorage *storage; + struct _ValidateNode *node; + int ret; + + curBuf = validateListIterator(&list->userBuffers); + + /* + * User-space validation callbacks. + */ + + while (curBuf) { + node = validateListNode(curBuf); + storage = (struct _WsbmBufStorage *)node->buf; + if (storage->pool->validate) { + ret = storage->pool->validate(storage, node->set_flags, + node->clr_flags); + if (ret) + return ret; + } + curBuf = validateListNext(&list->userBuffers, curBuf); + } + return 0; +} + +int +wsbmBOUnvalidateUserList(struct _WsbmBufferList *list) +{ + void *curBuf; + struct _WsbmBufStorage *storage; + struct _ValidateNode *node; + + curBuf = validateListIterator(&list->userBuffers); + + /* + * User-space validation callbacks. + */ + + while (curBuf) { + node = validateListNode(curBuf); + storage = (struct _WsbmBufStorage *)node->buf; + if (storage->pool->unvalidate) { + storage->pool->unvalidate(storage); + } + wsbmAtomicDec(&storage->onList); + wsbmBufStorageUnref(&storage); + curBuf = validateListNext(&list->userBuffers, curBuf); + } + return wsbmBOResetList(list); +} + +void +wsbmPoolTakeDown(struct _WsbmBufferPool *pool) +{ + pool->takeDown(pool); + +} + +unsigned long +wsbmBOSize(struct _WsbmBufferObject *buf) +{ + unsigned long size; + struct _WsbmBufStorage *storage; + + storage = buf->storage; + size = storage->pool->size(storage); + + return size; + +} + +struct _ValidateList * +wsbmGetKernelValidateList(struct _WsbmBufferList *list) +{ + return (list->hasKernelBuffers) ? &list->kernelBuffers : NULL; +} + +struct _ValidateList * +wsbmGetUserValidateList(struct _WsbmBufferList *list) +{ + return &list->userBuffers; +} + +struct _ValidateNode * +validateListNode(void *iterator) +{ + struct _WsbmListHead *l = (struct _WsbmListHead *)iterator; + + return WSBMLISTENTRY(l, struct _ValidateNode, head); +} + +void * +validateListIterator(struct _ValidateList *list) +{ + void *ret = list->list.next; + + if (ret == &list->list) + return NULL; + return ret; +} + +void * +validateListNext(struct _ValidateList *list, void *iterator) +{ + void *ret; + + struct _WsbmListHead *l = (struct _WsbmListHead *)iterator; + + ret = l->next; + if (ret == &list->list) + return NULL; + return ret; +} + +uint32_t +wsbmKBufHandle(const struct _WsbmKernelBuf * kBuf) +{ + return kBuf->handle; +} + +extern void +wsbmUpdateKBuf(struct _WsbmKernelBuf *kBuf, + uint64_t gpuOffset, uint32_t placement, + uint32_t fence_type_mask) +{ + kBuf->gpuOffset = gpuOffset; + kBuf->placement = placement; + kBuf->fence_type_mask = fence_type_mask; +} + +extern struct _WsbmKernelBuf * +wsbmKBuf(const struct _WsbmBufferObject *buf) +{ + struct _WsbmBufStorage *storage = buf->storage; + + return storage->pool->kernel(storage); +} diff --git a/src/wsbm_manager.h b/src/wsbm_manager.h new file mode 100644 index 0000000..ee0636f --- /dev/null +++ b/src/wsbm_manager.h @@ -0,0 +1,182 @@ +/************************************************************************** + * + * Copyright 2006-2008 Tungsten Graphics, Inc., Cedar Park, TX., USA + * All Rights Reserved. + * Copyright 2009 Vmware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellström + * Keith Whitwell + */ + +#ifndef _WSBM_MANAGER_H_ +#define _WSBM_MANAGER_H_ +#include "wsbm_fencemgr.h" +#include "wsbm_util.h" +#include "wsbm_driver.h" + +#define WSBM_VERSION_MAJOR 1 +#define WSBM_VERSION_MINOR 1 +#define WSBM_VERSION_PL 0 + +struct _WsbmFenceObject; +struct _WsbmBufferObject; +struct _WsbmBufferPool; +struct _WsbmBufferList; + +/* + * These flags mimics the TTM closely, but since + * this library is not dependant on TTM, we need to + * replicate them here, and if there is a discrepancy, + * that needs to be resolved in the buffer pool using + * the TTM flags. + */ + +#define WSBM_PL_MASK_MEM 0x0000FFFF + +#define WSBM_PL_FLAG_SYSTEM (1 << 0) +#define WSBM_PL_FLAG_TT (1 << 1) +#define WSBM_PL_FLAG_VRAM (1 << 2) +#define WSBM_PL_FLAG_PRIV0 (1 << 3) +#define WSBM_PL_FLAG_SWAPPED (1 << 15) +#define WSBM_PL_FLAG_CACHED (1 << 16) +#define WSBM_PL_FLAG_UNCACHED (1 << 17) +#define WSBM_PL_FLAG_WC (1 << 18) +#define WSBM_PL_FLAG_SHARED (1 << 20) +#define WSBM_PL_FLAG_NO_EVICT (1 << 21) + +#define WSBM_ACCESS_READ (1 << 0) +#define WSBM_ACCESS_WRITE (1 << 1) + +#define WSBM_SYNCCPU_READ WSBM_ACCESS_READ +#define WSBM_SYNCCPU_WRITE WSBM_ACCESS_WRITE +#define WSBM_SYNCCPU_DONT_BLOCK (1 << 2) +#define WSBM_SYNCCPU_TRY_CACHED (1 << 3) + +extern void *wsbmBOMap(struct _WsbmBufferObject *buf, unsigned mode); +extern void wsbmBOUnmap(struct _WsbmBufferObject *buf); +extern int wsbmBOSyncForCpu(struct _WsbmBufferObject *buf, unsigned mode); +extern void wsbmBOReleaseFromCpu(struct _WsbmBufferObject *buf, + unsigned mode); + +extern unsigned long wsbmBOOffsetHint(struct _WsbmBufferObject *buf); +extern unsigned long wsbmBOPoolOffset(struct _WsbmBufferObject *buf); + +extern uint32_t wsbmBOPlacementHint(struct _WsbmBufferObject *buf); +extern struct _WsbmBufferObject *wsbmBOReference(struct _WsbmBufferObject + *buf); +extern void wsbmBOUnreference(struct _WsbmBufferObject **p_buf); + +extern int wsbmBOData(struct _WsbmBufferObject *r_buf, + unsigned size, const void *data, + struct _WsbmBufferPool *pool, uint32_t placement); +extern int wsbmBOSetStatus(struct _WsbmBufferObject *buf, + uint32_t setPlacement, uint32_t clrPlacement); +extern int wsbmBOSubData(struct _WsbmBufferObject *buf, + unsigned long offset, unsigned long size, + const void *data, + int (*accelCopy) (struct _WsbmBufferObject *, + struct _WsbmBufferObject *)); +extern struct _WsbmBufferObject *wsbmBOClone(struct _WsbmBufferObject *buf, + int (*accelCopy) (struct + _WsbmBufferObject + *, + struct + _WsbmBufferObject + *)); + +extern int wsbmBOGetSubData(struct _WsbmBufferObject *buf, + unsigned long offset, unsigned long size, + void *data); +extern int wsbmGenBuffers(struct _WsbmBufferPool *pool, + unsigned n, + struct _WsbmBufferObject *buffers[], + unsigned alignment, uint32_t placement); + +struct _WsbmBufferObject *wsbmBOCreateSimple(struct _WsbmBufferPool *pool, + unsigned long size, + uint32_t placement, + unsigned alignment, + size_t extra_size, + size_t * offset); + +extern void wsbmDeleteBuffers(unsigned n, + struct _WsbmBufferObject *buffers[]); +extern struct _WsbmBufferList *wsbmBOCreateList(int target, + int hasKernelBuffers); +extern int wsbmBOResetList(struct _WsbmBufferList *list); +extern int wsbmBOAddListItem(struct _WsbmBufferList *list, + struct _WsbmBufferObject *buf, + uint64_t flags, uint64_t mask, int *itemLoc, + struct _ValidateNode **node); + +extern void wsbmBOFreeList(struct _WsbmBufferList *list); +extern int wsbmBOFenceUserList(struct _WsbmBufferList *list, + struct _WsbmFenceObject *fence); + +extern int wsbmBOUnrefUserList(struct _WsbmBufferList *list); +extern int wsbmBOValidateUserList(struct _WsbmBufferList *list); +extern int wsbmBOUnvalidateUserList(struct _WsbmBufferList *list); + +extern void wsbmBOFence(struct _WsbmBufferObject *buf, + struct _WsbmFenceObject *fence); + +extern void wsbmPoolTakeDown(struct _WsbmBufferPool *pool); +extern int wsbmBOSetReferenced(struct _WsbmBufferObject *buf, + unsigned long handle); +unsigned long wsbmBOSize(struct _WsbmBufferObject *buf); +extern void wsbmBOWaitIdle(struct _WsbmBufferObject *buf, int lazy); +extern int wsbmBOOnList(const struct _WsbmBufferObject *buf); + +extern void wsbmPoolTakeDown(struct _WsbmBufferPool *pool); + +extern void wsbmReadLockKernelBO(void); +extern void wsbmReadUnlockKernelBO(void); +extern void wsbmWriteLockKernelBO(void); +extern void wsbmWriteUnlockKernelBO(void); + +extern int wsbmInit(struct _WsbmThreadFuncs *tf, struct _WsbmVNodeFuncs *vf); +extern void wsbmTakedown(void); +extern int wsbmIsInitialized(void); +extern void wsbmCommonDataSet(void *d); +extern void *wsbmCommonDataGet(void); + +extern struct _ValidateList *wsbmGetKernelValidateList(struct _WsbmBufferList + *list); +extern struct _ValidateList *wsbmGetUserValidateList(struct _WsbmBufferList + *list); + +extern struct _ValidateNode *validateListNode(void *iterator); +extern void *validateListIterator(struct _ValidateList *list); +extern void *validateListNext(struct _ValidateList *list, void *iterator); + +extern uint32_t wsbmKBufHandle(const struct _WsbmKernelBuf *); +extern void wsbmUpdateKBuf(struct _WsbmKernelBuf *, + uint64_t gpuOffset, + uint32_t placement, uint32_t fence_flags); + +extern struct _WsbmKernelBuf *wsbmKBuf(const struct _WsbmBufferObject *buf); + +#endif diff --git a/src/wsbm_mm.c b/src/wsbm_mm.c new file mode 100644 index 0000000..88871ba --- /dev/null +++ b/src/wsbm_mm.c @@ -0,0 +1,289 @@ +/************************************************************************** + * + * Copyright 2006-2008 Tungsten Graphics, Inc., Cedar Park, TX., USA. + * All Rights Reserved. + * Copyright 2009 VMware, Inc., Palo Alto, CA., USA. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ + +/* + * Generic simple memory manager implementation. Intended to be used as a base + * class implementation for more advanced memory managers. + * + * Note that the algorithm used is quite simple and there might be substantial + * performance gains if a smarter free list is implemented. Currently it is just an + * unordered stack of free regions. This could easily be improved if an RB-tree + * is used instead. At least if we expect heavy fragmentation. + * Note that this implementation is more or less identical to the drm core manager + * in the linux kernel. + * + * Authors: + * Thomas Hellström + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "wsbm_mm.h" +#include +#include + +unsigned long +wsbmMMTailSpace(struct _WsbmMM *mm) +{ + struct _WsbmListHead *tail_node; + struct _WsbmMMNode *entry; + + tail_node = mm->ml_entry.prev; + entry = WSBMLISTENTRY(tail_node, struct _WsbmMMNode, ml_entry); + + if (!entry->free) + return 0; + + return entry->size; +} + +int +wsbmMMRemoveSpaceFromTail(struct _WsbmMM *mm, unsigned long size) +{ + struct _WsbmListHead *tail_node; + struct _WsbmMMNode *entry; + + tail_node = mm->ml_entry.prev; + entry = WSBMLISTENTRY(tail_node, struct _WsbmMMNode, ml_entry); + + if (!entry->free) + return -ENOMEM; + + if (entry->size <= size) + return -ENOMEM; + + entry->size -= size; + return 0; +} + +static int +wsbmMMCreateTailNode(struct _WsbmMM *mm, + unsigned long start, unsigned long size) +{ + struct _WsbmMMNode *child; + + child = (struct _WsbmMMNode *)malloc(sizeof(*child)); + if (!child) + return -ENOMEM; + + child->free = 1; + child->size = size; + child->start = start; + child->mm = mm; + + WSBMLISTADDTAIL(&child->ml_entry, &mm->ml_entry); + WSBMLISTADDTAIL(&child->fl_entry, &mm->fl_entry); + + return 0; +} + +static struct _WsbmMMNode * +wsbmMMSplitAtStart(struct _WsbmMMNode *parent, unsigned long size) +{ + struct _WsbmMMNode *child; + + child = (struct _WsbmMMNode *)malloc(sizeof(*child)); + if (!child) + return NULL; + + WSBMINITLISTHEAD(&child->fl_entry); + + child->free = 0; + child->size = size; + child->start = parent->start; + child->mm = parent->mm; + + WSBMLISTADDTAIL(&child->ml_entry, &parent->ml_entry); + WSBMINITLISTHEAD(&child->fl_entry); + + parent->size -= size; + parent->start += size; + return child; +} + +struct _WsbmMMNode * +wsbmMMGetBlock(struct _WsbmMMNode *parent, + unsigned long size, unsigned alignment) +{ + + struct _WsbmMMNode *align_splitoff = NULL; + struct _WsbmMMNode *child; + unsigned tmp = 0; + + if (alignment) + tmp = parent->start % alignment; + + if (tmp) { + align_splitoff = wsbmMMSplitAtStart(parent, alignment - tmp); + if (!align_splitoff) + return NULL; + } + + if (parent->size == size) { + WSBMLISTDELINIT(&parent->fl_entry); + parent->free = 0; + return parent; + } else { + child = wsbmMMSplitAtStart(parent, size); + } + + if (align_splitoff) + wsbmMMPutBlock(align_splitoff); + + return child; +} + +/* + * Put a block. Merge with the previous and / or next block if they are free. + * Otherwise add to the free stack. + */ + +void +wsbmMMPutBlock(struct _WsbmMMNode *cur) +{ + + struct _WsbmMM *mm = cur->mm; + struct _WsbmListHead *cur_head = &cur->ml_entry; + struct _WsbmListHead *root_head = &mm->ml_entry; + struct _WsbmMMNode *prev_node = NULL; + struct _WsbmMMNode *next_node; + + int merged = 0; + + if (cur_head->prev != root_head) { + prev_node = + WSBMLISTENTRY(cur_head->prev, struct _WsbmMMNode, ml_entry); + if (prev_node->free) { + prev_node->size += cur->size; + merged = 1; + } + } + if (cur_head->next != root_head) { + next_node = + WSBMLISTENTRY(cur_head->next, struct _WsbmMMNode, ml_entry); + if (next_node->free) { + if (merged) { + prev_node->size += next_node->size; + WSBMLISTDEL(&next_node->ml_entry); + WSBMLISTDEL(&next_node->fl_entry); + free(next_node); + } else { + next_node->size += cur->size; + next_node->start = cur->start; + merged = 1; + } + } + } + if (!merged) { + cur->free = 1; + WSBMLISTADD(&cur->fl_entry, &mm->fl_entry); + } else { + WSBMLISTDEL(&cur->ml_entry); + free(cur); + } +} + +struct _WsbmMMNode * +wsbmMMSearchFree(const struct _WsbmMM *mm, + unsigned long size, unsigned alignment, int best_match) +{ + struct _WsbmListHead *list; + const struct _WsbmListHead *free_stack = &mm->fl_entry; + struct _WsbmMMNode *entry; + struct _WsbmMMNode *best; + unsigned long best_size; + unsigned wasted; + + best = NULL; + best_size = ~0UL; + + WSBMLISTFOREACH(list, free_stack) { + entry = WSBMLISTENTRY(list, struct _WsbmMMNode, fl_entry); + + wasted = 0; + + if (entry->size < size) + continue; + + if (alignment) { + register unsigned tmp = entry->start % alignment; + + if (tmp) + wasted += alignment - tmp; + } + + if (entry->size >= size + wasted) { + if (!best_match) + return entry; + if (size < best_size) { + best = entry; + best_size = entry->size; + } + } + } + + return best; +} + +int +wsbmMMclean(struct _WsbmMM *mm) +{ + struct _WsbmListHead *head = &mm->ml_entry; + + return (head->next->next == head); +} + +int +wsbmMMinit(struct _WsbmMM *mm, unsigned long start, unsigned long size) +{ + WSBMINITLISTHEAD(&mm->ml_entry); + WSBMINITLISTHEAD(&mm->fl_entry); + + return wsbmMMCreateTailNode(mm, start, size); +} + +void +wsbmMMtakedown(struct _WsbmMM *mm) +{ + struct _WsbmListHead *bnode = mm->fl_entry.next; + struct _WsbmMMNode *entry; + + entry = WSBMLISTENTRY(bnode, struct _WsbmMMNode, fl_entry); + + if (entry->ml_entry.next != &mm->ml_entry || + entry->fl_entry.next != &mm->fl_entry) { + return; + } + + WSBMLISTDEL(&entry->fl_entry); + WSBMLISTDEL(&entry->ml_entry); + free(entry); +} diff --git a/src/wsbm_mm.h b/src/wsbm_mm.h new file mode 100644 index 0000000..d3bbe94 --- /dev/null +++ b/src/wsbm_mm.h @@ -0,0 +1,75 @@ +/************************************************************************** + * + * Copyright 2006-2008 Tungsten Graphics, Inc., Cedar Park, TX., USA. + * All Rights Reserved. + * Copyright 2009 VMware, Inc., Palo Alto, CA., USA. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ + +/* + * Generic simple memory manager implementation. Intended to be used as a base + * class implementation for more advanced memory managers. + * + * Note that the algorithm used is quite simple and there might be substantial + * performance gains if a smarter free list is implemented. Currently it is just an + * unordered stack of free regions. This could easily be improved if an RB-tree + * is used instead. At least if we expect heavy fragmentation. + * + * Authors: + * Thomas Hellstrom + */ + +#ifndef _WSBM_MM_H_ +#define _WSBM_MM_H_ + +#include "wsbm_util.h" +struct _WsbmMM +{ + struct _WsbmListHead fl_entry; + struct _WsbmListHead ml_entry; +}; + +struct _WsbmMMNode +{ + struct _WsbmListHead fl_entry; + struct _WsbmListHead ml_entry; + int free; + unsigned long start; + unsigned long size; + struct _WsbmMM *mm; +}; + +extern struct _WsbmMMNode *wsbmMMSearchFree(const struct _WsbmMM *mm, + unsigned long size, + unsigned alignment, + int best_match); +extern struct _WsbmMMNode *wsbmMMGetBlock(struct _WsbmMMNode *parent, + unsigned long size, + unsigned alignment); +extern void wsbmMMPutBlock(struct _WsbmMMNode *cur); +extern void wsbmMMtakedown(struct _WsbmMM *mm); +extern int wsbmMMinit(struct _WsbmMM *mm, unsigned long start, + unsigned long size); +extern int wsbmMMclean(struct _WsbmMM *mm); +#endif diff --git a/src/wsbm_pool.h b/src/wsbm_pool.h new file mode 100644 index 0000000..1d07523 --- /dev/null +++ b/src/wsbm_pool.h @@ -0,0 +1,161 @@ +/************************************************************************** + * + * Copyright 2006-2008 Tungsten Graphics, Inc., Cedar Park, TX., USA + * All Rights Reserved. + * Copyright 2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellström + */ + +#ifndef _WSBM_BUFPOOL_H_ +#define _WSBM_BUFPOOL_H_ + +#include +#include "wsbm_util.h" +#include "wsbm_driver.h" +#include "wsbm_atomic.h" + +struct _WsbmFenceObject; + +struct _WsbmBufStorage +{ + struct _WsbmBufferPool *pool; + struct _WsbmMutex mutex; + struct _WsbmAtomic refCount; + struct _WsbmAtomic onList; + void *destroyArg; + void (*destroyContainer) (void *); +}; + +struct _WsbmKernelBuf; + +struct _WsbmBufferPool +{ + int fd; + int (*map) (struct _WsbmBufStorage * buf, unsigned mode, void **virtual); + void (*unmap) (struct _WsbmBufStorage * buf); + int (*syncforcpu) (struct _WsbmBufStorage * buf, unsigned mode); + void (*releasefromcpu) (struct _WsbmBufStorage * buf, unsigned mode); + void (*destroy) (struct _WsbmBufStorage ** buf); + unsigned long (*offset) (struct _WsbmBufStorage * buf); + unsigned long (*poolOffset) (struct _WsbmBufStorage * buf); + uint32_t(*placement) (struct _WsbmBufStorage * buf); + unsigned long (*size) (struct _WsbmBufStorage * buf); + struct _WsbmKernelBuf *(*kernel) (struct _WsbmBufStorage * buf); + struct _WsbmBufStorage *(*create) (struct _WsbmBufferPool * pool, + unsigned long size, + uint32_t placement, + unsigned alignment); + struct _WsbmBufStorage *(*createByReference) (struct _WsbmBufferPool * + pool, uint32_t handle); + void (*fence) (struct _WsbmBufStorage * buf, + struct _WsbmFenceObject * fence); + void (*unvalidate) (struct _WsbmBufStorage * buf); + int (*validate) (struct _WsbmBufStorage * buf, uint64_t set_flags, + uint64_t clr_flags); + int (*waitIdle) (struct _WsbmBufStorage * buf, int lazy); + int (*setStatus) (struct _WsbmBufStorage * buf, + uint32_t set_placement, uint32_t clr_placement); + void (*takeDown) (struct _WsbmBufferPool * pool); +}; + +static inline int +wsbmBufStorageInit(struct _WsbmBufStorage *storage, + struct _WsbmBufferPool *pool) +{ + int ret = WSBM_MUTEX_INIT(&storage->mutex); + + if (ret) + return -ENOMEM; + storage->pool = pool; + wsbmAtomicSet(&storage->refCount, 1); + wsbmAtomicSet(&storage->onList, 0); + storage->destroyContainer = NULL; + return 0; +} + +static inline void +wsbmBufStorageTakedown(struct _WsbmBufStorage *storage) +{ + WSBM_MUTEX_FREE(&storage->mutex); +} + +static inline void +wsbmBufStorageUnref(struct _WsbmBufStorage **pStorage) +{ + struct _WsbmBufStorage *storage = *pStorage; + + *pStorage = NULL; + if (storage == NULL) + return; + + if (wsbmAtomicDecZero(&storage->refCount)) { + if (storage->destroyContainer) + storage->destroyContainer(storage->destroyArg); + storage->pool->destroy(&storage); + return; + } +} + +/* + * Builtin pools. + */ + +/* + * Kernel buffer objects. Size in multiples of page size. Page size aligned. + */ + +extern struct _WsbmBufferPool *wsbmTTMPoolInit(int fd, + unsigned int devOffset); +extern struct _WsbmBufferPool *wsbmMallocPoolInit(void); + +struct _WsbmSlabCache; +extern struct _WsbmBufferPool *wsbmSlabPoolInit(int fd, uint32_t devOffset, + uint32_t placement, + uint32_t validMask, + uint32_t smallestSize, + uint32_t numSizes, + uint32_t desiredNumBuffers, + uint32_t maxSlabSize, + uint32_t pageAlignment, + struct _WsbmSlabCache *cache); +extern struct _WsbmSlabCache *wsbmSlabCacheInit(uint32_t checkIntervalMsec, + uint32_t slabTimeoutMsec); +extern void wsbmSlabCacheFinish(struct _WsbmSlabCache *cache); + +extern struct _WsbmBufferPool *wsbmUserPoolInit(void *vramAddr, + unsigned long vramStart, + unsigned long vramSize, + void *agpAddr, + unsigned long agpStart, + unsigned long agpSize, + uint32_t(*fenceTypes) + (uint64_t set_flags)); + +extern void wsbmUserPoolClean(struct _WsbmBufferPool *pool, + int cleanVram, int cleanAgp); + +#endif diff --git a/src/wsbm_priv.h b/src/wsbm_priv.h new file mode 100644 index 0000000..89e4313 --- /dev/null +++ b/src/wsbm_priv.h @@ -0,0 +1,45 @@ +/************************************************************************** + * + * Copyright 2008 Tungsten Graphics, Inc., Cedar Park, TX., USA + * All Rights Reserved. + * Copyright 2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: + * Thomas Hellström + */ + +#ifndef _WSBM_PRIV_H_ +#define _WSBM_PRIV_H_ + +struct _WsbmKernelBuf +{ + uint64_t gpuOffset; + uint32_t handle; + uint32_t placement; + uint32_t fence_type_mask; +}; + +#endif diff --git a/src/wsbm_slabpool.c b/src/wsbm_slabpool.c new file mode 100644 index 0000000..51a85d0 --- /dev/null +++ b/src/wsbm_slabpool.c @@ -0,0 +1,1203 @@ +/************************************************************************** + * + * Copyright 2006-2008 Tungsten Graphics, Inc., Cedar Park, TX., USA + * All Rights Reserved. + * Copyright 2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "wsbm_pool.h" +#include "wsbm_fencemgr.h" +#include "wsbm_priv.h" +#include "wsbm_manager.h" + +#define WSBM_SLABPOOL_ALLOC_RETRIES 100 +#define DRMRESTARTCOMMANDWRITE(_fd, _val, _arg, _ret) \ + do { \ + (_ret) = drmCommandWrite(_fd, _val, &(_arg), sizeof(_arg)); \ + } while ((_ret) == -EAGAIN || (_ret) == -ERESTART); \ + +#define DRMRESTARTCOMMANDWRITEREAD(_fd, _val, _arg, _ret) \ + do { \ + (_ret) = drmCommandWriteRead(_fd, _val, &(_arg), sizeof(_arg)); \ + } while ((_ret) == -EAGAIN || (_ret) == -ERESTART); \ + + +#ifdef DEBUG_FENCESIGNALED +static int createbuffer = 0; +static int fencesignaled = 0; +#endif + +struct _WsbmSlab; + +struct _WsbmSlabBuffer +{ + struct _WsbmKernelBuf kBuf; + struct _WsbmBufStorage storage; + struct _WsbmCond event; + + /* + * Remains constant after creation. + */ + + int isSlabBuffer; + struct _WsbmSlab *parent; + uint32_t start; + void *virtual; + unsigned long requestedSize; + uint64_t mapHandle; + + /* + * Protected by struct _WsbmSlabSizeHeader::mutex + */ + + struct _WsbmListHead head; + + /* + * Protected by this::mutex + */ + + struct _WsbmFenceObject *fence; + uint32_t fenceType; + struct _WsbmAtomic writers; /* (Only upping) */ + int unFenced; +}; + +struct _WsbmSlabPool; +struct _WsbmSlabKernelBO +{ + + /* + * Constant at creation + */ + + struct _WsbmKernelBuf kBuf; + uint32_t pageAlignment; + void *virtual; + unsigned long actualSize; + uint64_t mapHandle; + + /* + * Protected by struct _WsbmSlabCache::mutex + */ + + struct _WsbmSlabPool *slabPool; + uint32_t proposedPlacement; + struct _WsbmListHead timeoutHead; + struct _WsbmListHead head; + struct timeval timeFreed; +}; + +struct _WsbmSlab +{ + struct _WsbmListHead head; + struct _WsbmListHead freeBuffers; + uint32_t numBuffers; + uint32_t numFree; + struct _WsbmSlabBuffer *buffers; + struct _WsbmSlabSizeHeader *header; + struct _WsbmSlabKernelBO *kbo; +}; + +struct _WsbmSlabSizeHeader +{ + /* + * Constant at creation. + */ + struct _WsbmSlabPool *slabPool; + uint32_t bufSize; + + /* + * Protected by this::mutex + */ + + struct _WsbmListHead slabs; + struct _WsbmListHead freeSlabs; + struct _WsbmListHead delayedBuffers; + uint32_t numDelayed; + struct _WsbmMutex mutex; +}; + +struct _WsbmSlabCache +{ + struct timeval slabTimeout; + struct timeval checkInterval; + struct timeval nextCheck; + struct _WsbmListHead timeoutList; + struct _WsbmListHead unCached; + struct _WsbmListHead cached; + struct _WsbmMutex mutex; +}; + +struct _WsbmSlabPool +{ + struct _WsbmBufferPool pool; + + /* + * The data of this structure remains constant after + * initialization and thus needs no mutex protection. + */ + + unsigned int devOffset; + struct _WsbmSlabCache *cache; + uint32_t proposedPlacement; + uint32_t validMask; + uint32_t *bucketSizes; + uint32_t numBuckets; + uint32_t pageSize; + int pageAlignment; + int maxSlabSize; + int desiredNumBuffers; + struct _WsbmSlabSizeHeader *headers; +}; + +static inline struct _WsbmSlabPool * +slabPoolFromPool(struct _WsbmBufferPool *pool) +{ + return containerOf(pool, struct _WsbmSlabPool, pool); +} + +static inline struct _WsbmSlabPool * +slabPoolFromBuf(struct _WsbmSlabBuffer *sBuf) +{ + return slabPoolFromPool(sBuf->storage.pool); +} + +static inline struct _WsbmSlabBuffer * +slabBuffer(struct _WsbmBufStorage *buf) +{ + return containerOf(buf, struct _WsbmSlabBuffer, storage); +} + +/* + * FIXME: Perhaps arrange timeout slabs in size buckets for fast + * retreival?? + */ + +static inline int +wsbmTimeAfterEq(struct timeval *arg1, struct timeval *arg2) +{ + return ((arg1->tv_sec > arg2->tv_sec) || + ((arg1->tv_sec == arg2->tv_sec) && + (arg1->tv_usec > arg2->tv_usec))); +} + +static inline void +wsbmTimeAdd(struct timeval *arg, struct timeval *add) +{ + unsigned int sec; + + arg->tv_sec += add->tv_sec; + arg->tv_usec += add->tv_usec; + sec = arg->tv_usec / 1000000; + arg->tv_sec += sec; + arg->tv_usec -= sec * 1000000; +} + +static void +wsbmFreeKernelBO(struct _WsbmSlabKernelBO *kbo) +{ + struct ttm_pl_reference_req arg; + struct _WsbmSlabPool *slabPool; + + if (!kbo) + return; + + slabPool = kbo->slabPool; + arg.handle = kbo->kBuf.handle; + (void)munmap(kbo->virtual, kbo->actualSize); + (void)drmCommandWrite(slabPool->pool.fd, + slabPool->devOffset + TTM_PL_UNREF, &arg, + sizeof(arg)); + free(kbo); +} + +static void +wsbmFreeTimeoutKBOsLocked(struct _WsbmSlabCache *cache, struct timeval *time) +{ + struct _WsbmListHead *list, *next; + struct _WsbmSlabKernelBO *kbo; + + if (!wsbmTimeAfterEq(time, &cache->nextCheck)) + return; + + WSBMLISTFOREACHSAFE(list, next, &cache->timeoutList) { + kbo = WSBMLISTENTRY(list, struct _WsbmSlabKernelBO, timeoutHead); + + if (!wsbmTimeAfterEq(time, &kbo->timeFreed)) + break; + + WSBMLISTDELINIT(&kbo->timeoutHead); + WSBMLISTDELINIT(&kbo->head); + wsbmFreeKernelBO(kbo); + } + + cache->nextCheck = *time; + wsbmTimeAdd(&cache->nextCheck, &cache->checkInterval); +} + +/* + * Add a _SlabKernelBO to the free slab manager. + * This means that it is available for reuse, but if it's not + * reused in a while, it will be freed. + */ + +static void +wsbmSetKernelBOFree(struct _WsbmSlabCache *cache, + struct _WsbmSlabKernelBO *kbo) +{ + struct timeval time; + struct timeval timeFreed; + + gettimeofday(&time, NULL); + timeFreed = time; + WSBM_MUTEX_LOCK(&cache->mutex); + wsbmTimeAdd(&timeFreed, &cache->slabTimeout); + kbo->timeFreed = timeFreed; + + if (kbo->kBuf.placement & TTM_PL_FLAG_CACHED) + WSBMLISTADD(&kbo->head, &cache->cached); + else + WSBMLISTADD(&kbo->head, &cache->unCached); + + WSBMLISTADDTAIL(&kbo->timeoutHead, &cache->timeoutList); + wsbmFreeTimeoutKBOsLocked(cache, &time); + + WSBM_MUTEX_UNLOCK(&cache->mutex); +} + +/* + * Get a _SlabKernelBO for us to use as storage for a slab. + */ + +static struct _WsbmSlabKernelBO * +wsbmAllocKernelBO(struct _WsbmSlabSizeHeader *header) +{ + struct _WsbmSlabPool *slabPool = header->slabPool; + struct _WsbmSlabCache *cache = slabPool->cache; + struct _WsbmListHead *list, *head; + uint32_t size = header->bufSize * slabPool->desiredNumBuffers; + struct _WsbmSlabKernelBO *kbo; + struct _WsbmSlabKernelBO *kboTmp; + int ret; + + /* + * FIXME: We should perhaps allow some variation in slabsize in order + * to efficiently reuse slabs. + */ + + size = (size <= slabPool->maxSlabSize) ? size : slabPool->maxSlabSize; + if (size < header->bufSize) + size = header->bufSize; + size = (size + slabPool->pageSize - 1) & ~(slabPool->pageSize - 1); + WSBM_MUTEX_LOCK(&cache->mutex); + + kbo = NULL; + + retry: + head = (slabPool->proposedPlacement & TTM_PL_FLAG_CACHED) ? + &cache->cached : &cache->unCached; + + WSBMLISTFOREACH(list, head) { + kboTmp = WSBMLISTENTRY(list, struct _WsbmSlabKernelBO, head); + + if ((kboTmp->actualSize == size) && + (slabPool->pageAlignment == 0 || + (kboTmp->pageAlignment % slabPool->pageAlignment) == 0)) { + + if (!kbo) + kbo = kboTmp; + + if ((kbo->proposedPlacement ^ slabPool->proposedPlacement) == 0) + break; + + } + } + + if (kbo) { + WSBMLISTDELINIT(&kbo->head); + WSBMLISTDELINIT(&kbo->timeoutHead); + } + + WSBM_MUTEX_UNLOCK(&cache->mutex); + + if (kbo) { + uint32_t new_mask = + kbo->proposedPlacement ^ slabPool->proposedPlacement; + + ret = 0; + if (new_mask) { + union ttm_pl_setstatus_arg arg; + struct ttm_pl_setstatus_req *req = &arg.req; + struct ttm_pl_rep *rep = &arg.rep; + + req->handle = kbo->kBuf.handle; + req->set_placement = slabPool->proposedPlacement & new_mask; + req->clr_placement = ~slabPool->proposedPlacement & new_mask; + DRMRESTARTCOMMANDWRITEREAD(slabPool->pool.fd, + slabPool->devOffset + TTM_PL_SETSTATUS, + arg, ret); + if (ret == 0) { + kbo->kBuf.gpuOffset = rep->gpu_offset; + kbo->kBuf.placement = rep->placement; + } + kbo->proposedPlacement = slabPool->proposedPlacement; + } + + if (ret == 0) + return kbo; + + wsbmFreeKernelBO(kbo); + kbo = NULL; + goto retry; + } + + kbo = calloc(1, sizeof(*kbo)); + if (!kbo) + return NULL; + + { + union ttm_pl_create_arg arg; + + kbo->slabPool = slabPool; + WSBMINITLISTHEAD(&kbo->head); + WSBMINITLISTHEAD(&kbo->timeoutHead); + + arg.req.size = size; + arg.req.placement = slabPool->proposedPlacement; + arg.req.page_alignment = slabPool->pageAlignment; + + DRMRESTARTCOMMANDWRITEREAD(slabPool->pool.fd, + slabPool->devOffset + TTM_PL_CREATE, + arg, ret); + if (ret) + goto out_err0; + + kbo->kBuf.gpuOffset = arg.rep.gpu_offset; + kbo->kBuf.placement = arg.rep.placement; + kbo->kBuf.handle = arg.rep.handle; + + kbo->actualSize = arg.rep.bo_size; + kbo->mapHandle = arg.rep.map_handle; + kbo->proposedPlacement = slabPool->proposedPlacement; + } + + kbo->virtual = mmap(0, kbo->actualSize, + PROT_READ | PROT_WRITE, MAP_SHARED, + slabPool->pool.fd, kbo->mapHandle); + + if (kbo->virtual == MAP_FAILED) { + ret = -errno; + goto out_err1; + } + + return kbo; + + out_err1: + { + struct ttm_pl_reference_req arg = {.handle = kbo->kBuf.handle }; + + (void)drmCommandWrite(slabPool->pool.fd, + slabPool->devOffset + TTM_PL_UNREF, + &arg, sizeof(arg)); + } + out_err0: + free(kbo); + return NULL; +} + +static int +wsbmAllocSlab(struct _WsbmSlabSizeHeader *header) +{ + struct _WsbmSlab *slab; + struct _WsbmSlabBuffer *sBuf; + uint32_t numBuffers; + int ret; + int i; + + slab = calloc(1, sizeof(*slab)); + if (!slab) + return -ENOMEM; + + slab->kbo = wsbmAllocKernelBO(header); + if (!slab->kbo) { + ret = -ENOMEM; + goto out_err0; + } + + numBuffers = slab->kbo->actualSize / header->bufSize; + + slab->buffers = calloc(numBuffers, sizeof(*slab->buffers)); + if (!slab->buffers) { + ret = -ENOMEM; + goto out_err1; + } + + WSBMINITLISTHEAD(&slab->head); + WSBMINITLISTHEAD(&slab->freeBuffers); + slab->numBuffers = numBuffers; + slab->numFree = 0; + slab->header = header; + + sBuf = slab->buffers; + for (i = 0; i < numBuffers; ++i) { + ret = wsbmBufStorageInit(&sBuf->storage, &header->slabPool->pool); + if (ret) + goto out_err2; + sBuf->parent = slab; + sBuf->start = i * header->bufSize; + sBuf->virtual = (void *)((uint8_t *) slab->kbo->virtual + + sBuf->start); + wsbmAtomicSet(&sBuf->writers, 0); + sBuf->isSlabBuffer = 1; + WSBM_COND_INIT(&sBuf->event); + WSBMLISTADDTAIL(&sBuf->head, &slab->freeBuffers); + slab->numFree++; + sBuf++; + } + + WSBMLISTADDTAIL(&slab->head, &header->slabs); + + return 0; + + out_err2: + sBuf = slab->buffers; + for (i = 0; i < numBuffers; ++i) { + if (sBuf->parent == slab) { + WSBM_COND_FREE(&sBuf->event); + wsbmBufStorageTakedown(&sBuf->storage); + } + sBuf++; + } + free(slab->buffers); + out_err1: + wsbmSetKernelBOFree(header->slabPool->cache, slab->kbo); + out_err0: + free(slab); + return ret; +} + +/* + * Delete a buffer from the slab header delayed list and put + * it on the slab free list. + */ + +static void +wsbmSlabFreeBufferLocked(struct _WsbmSlabBuffer *buf) +{ + struct _WsbmSlab *slab = buf->parent; + struct _WsbmSlabSizeHeader *header = slab->header; + struct _WsbmListHead *list = &buf->head; + + WSBMLISTDEL(list); + WSBMLISTADDTAIL(list, &slab->freeBuffers); + slab->numFree++; + + if (slab->head.next == &slab->head) + WSBMLISTADDTAIL(&slab->head, &header->slabs); + + if (slab->numFree == slab->numBuffers) { + list = &slab->head; + WSBMLISTDEL(list); + WSBMLISTADDTAIL(list, &header->freeSlabs); + } + + if (header->slabs.next == &header->slabs || + slab->numFree != slab->numBuffers) { + + struct _WsbmListHead *next; + struct _WsbmSlabCache *cache = header->slabPool->cache; + + WSBMLISTFOREACHSAFE(list, next, &header->freeSlabs) { + int i; + struct _WsbmSlabBuffer *sBuf; + + slab = WSBMLISTENTRY(list, struct _WsbmSlab, head); + + WSBMLISTDELINIT(list); + + sBuf = slab->buffers; + for (i = 0; i < slab->numBuffers; ++i) { + if (sBuf->parent == slab) { + WSBM_COND_FREE(&sBuf->event); + wsbmBufStorageTakedown(&sBuf->storage); + } + sBuf++; + } + wsbmSetKernelBOFree(cache, slab->kbo); + free(slab->buffers); + free(slab); + } + } +} + +static void +wsbmSlabCheckFreeLocked(struct _WsbmSlabSizeHeader *header, int wait) +{ + struct _WsbmListHead *list, *prev, *first, *head; + struct _WsbmSlabBuffer *sBuf; + struct _WsbmSlab *slab; + int firstWasSignaled = 1; + int signaled; + int i; + int ret; + + /* + * Rerun the freeing test if the youngest tested buffer + * was signaled, since there might be more idle buffers + * in the delay list. + */ + + while (firstWasSignaled) { + firstWasSignaled = 0; + signaled = 0; + first = header->delayedBuffers.next; + + /* Only examine the oldest 1/3 of delayed buffers: + */ + if (header->numDelayed > 3) { + for (i = 0; i < header->numDelayed; i += 3) { + first = first->next; + } + } + + /* + * No need to take the buffer mutex for each buffer we loop + * through since we're currently the only user. + */ + + head = first->next; + WSBMLISTFOREACHPREVSAFE(list, prev, head) { + + if (list == &header->delayedBuffers) + break; + + sBuf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head); + + slab = sBuf->parent; + + if (!signaled) { + if (wait) { + ret = wsbmFenceFinish(sBuf->fence, sBuf->fenceType, 0); + if (ret) + break; + signaled = 1; + wait = 0; + } else { + signaled = + wsbmFenceSignaled(sBuf->fence, sBuf->fenceType); +#ifdef DEBUG_FENCESIGNALED + fencesignaled++; +#endif + } + if (signaled) { + if (list == first) + firstWasSignaled = 1; + wsbmFenceUnreference(&sBuf->fence); + header->numDelayed--; + wsbmSlabFreeBufferLocked(sBuf); + } else + break; + } else if (wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType)) { + wsbmFenceUnreference(&sBuf->fence); + header->numDelayed--; + wsbmSlabFreeBufferLocked(sBuf); + } + } + } +} + +static struct _WsbmSlabBuffer * +wsbmSlabAllocBuffer(struct _WsbmSlabSizeHeader *header) +{ + static struct _WsbmSlabBuffer *buf; + struct _WsbmSlab *slab; + struct _WsbmListHead *list; + int count = WSBM_SLABPOOL_ALLOC_RETRIES; + + WSBM_MUTEX_LOCK(&header->mutex); + while (header->slabs.next == &header->slabs && count > 0) { + wsbmSlabCheckFreeLocked(header, 0); + if (header->slabs.next != &header->slabs) + break; + + WSBM_MUTEX_UNLOCK(&header->mutex); + if (count != WSBM_SLABPOOL_ALLOC_RETRIES) + usleep(1000); + WSBM_MUTEX_LOCK(&header->mutex); + (void)wsbmAllocSlab(header); + count--; + } + + list = header->slabs.next; + if (list == &header->slabs) { + WSBM_MUTEX_UNLOCK(&header->mutex); + return NULL; + } + slab = WSBMLISTENTRY(list, struct _WsbmSlab, head); + if (--slab->numFree == 0) + WSBMLISTDELINIT(list); + + list = slab->freeBuffers.next; + WSBMLISTDELINIT(list); + + WSBM_MUTEX_UNLOCK(&header->mutex); + buf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head); + + buf->storage.destroyContainer = NULL; + +#ifdef DEBUG_FENCESIGNALED + createbuffer++; +#endif + return buf; +} + +static struct _WsbmBufStorage * +pool_create(struct _WsbmBufferPool *pool, unsigned long size, + uint32_t placement, unsigned alignment) +{ + struct _WsbmSlabPool *slabPool = slabPoolFromPool(pool); + struct _WsbmSlabSizeHeader *header; + struct _WsbmSlabBuffer *sBuf; + int i; + int ret; + + /* + * FIXME: Check for compatibility. + */ + + header = slabPool->headers; + for (i = 0; i < slabPool->numBuckets; ++i) { + if (header->bufSize >= size) + break; + header++; + } + + if (i < slabPool->numBuckets) { + sBuf = wsbmSlabAllocBuffer(header); + return ((sBuf) ? &sBuf->storage : NULL); + } + + /* + * Fall back to allocate a buffer object directly from DRM. + * and wrap it in a wsbmBO structure. + */ + + sBuf = calloc(1, sizeof(*sBuf)); + + if (!sBuf) + return NULL; + + if (alignment) { + if ((alignment < slabPool->pageSize) + && (slabPool->pageSize % alignment)) + goto out_err0; + if ((alignment > slabPool->pageSize) + && (alignment % slabPool->pageSize)) + goto out_err0; + } + + ret = wsbmBufStorageInit(&sBuf->storage, pool); + if (ret) + goto out_err0; + + ret = WSBM_COND_INIT(&sBuf->event); + if (ret) + goto out_err1; + + { + union ttm_pl_create_arg arg; + + arg.req.size = size; + arg.req.placement = placement; + arg.req.page_alignment = alignment / slabPool->pageSize; + + DRMRESTARTCOMMANDWRITEREAD(pool->fd, + slabPool->devOffset + TTM_PL_CREATE, + arg, ret); + + if (ret) + goto out_err2; + + sBuf->kBuf.gpuOffset = arg.rep.gpu_offset; + sBuf->kBuf.placement = arg.rep.placement; + sBuf->kBuf.handle = arg.rep.handle; + sBuf->mapHandle = arg.rep.map_handle; + sBuf->requestedSize = size; + + sBuf->virtual = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, + pool->fd, sBuf->mapHandle); + + if (sBuf->virtual == MAP_FAILED) + goto out_err3; + } + + wsbmAtomicSet(&sBuf->writers, 0); + return &sBuf->storage; + out_err3: + { + struct ttm_pl_reference_req arg; + + arg.handle = sBuf->kBuf.handle; + (void)drmCommandWriteRead(pool->fd, + slabPool->devOffset + TTM_PL_UNREF, + &arg, sizeof(arg)); + } + out_err2: + WSBM_COND_FREE(&sBuf->event); + out_err1: + wsbmBufStorageTakedown(&sBuf->storage); + out_err0: + free(sBuf); + return NULL; +} + +static void +pool_destroy(struct _WsbmBufStorage **p_buf) +{ + struct _WsbmBufStorage *buf = *p_buf; + struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); + struct _WsbmSlab *slab; + struct _WsbmSlabSizeHeader *header; + + *p_buf = NULL; + + if (!sBuf->isSlabBuffer) { + struct _WsbmSlabPool *slabPool = slabPoolFromBuf(sBuf); + struct ttm_pl_reference_req arg; + + if (sBuf->virtual != NULL) { + (void)munmap(sBuf->virtual, sBuf->requestedSize); + sBuf->virtual = NULL; + } + + arg.handle = sBuf->kBuf.handle; + (void)drmCommandWrite(slabPool->pool.fd, + slabPool->devOffset + TTM_PL_UNREF, + &arg, sizeof(arg)); + + WSBM_COND_FREE(&sBuf->event); + wsbmBufStorageTakedown(&sBuf->storage); + free(sBuf); + return; + } + + slab = sBuf->parent; + header = slab->header; + + /* + * No need to take the buffer mutex below since we're the only user. + */ + + WSBM_MUTEX_LOCK(&header->mutex); + sBuf->unFenced = 0; + wsbmAtomicSet(&sBuf->writers, 0); + wsbmAtomicSet(&sBuf->storage.refCount, 1); + + if (sBuf->fence && !wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType)) { + WSBMLISTADDTAIL(&sBuf->head, &header->delayedBuffers); + header->numDelayed++; + } else { + if (sBuf->fence) + wsbmFenceUnreference(&sBuf->fence); + wsbmSlabFreeBufferLocked(sBuf); + } + + WSBM_MUTEX_UNLOCK(&header->mutex); +} + +static void +waitIdleLocked(struct _WsbmSlabBuffer *sBuf, int lazy) +{ + struct _WsbmBufStorage *storage = &sBuf->storage; + + while (sBuf->unFenced || sBuf->fence != NULL) { + + if (sBuf->unFenced) + WSBM_COND_WAIT(&sBuf->event, &storage->mutex); + + if (sBuf->fence != NULL) { + if (!wsbmFenceSignaled(sBuf->fence, sBuf->fenceType)) { + struct _WsbmFenceObject *fence = + wsbmFenceReference(sBuf->fence); + + WSBM_MUTEX_UNLOCK(&storage->mutex); + (void)wsbmFenceFinish(fence, sBuf->fenceType, lazy); + WSBM_MUTEX_LOCK(&storage->mutex); + if (sBuf->fence == fence) + wsbmFenceUnreference(&sBuf->fence); + + wsbmFenceUnreference(&fence); + } else { + wsbmFenceUnreference(&sBuf->fence); + } + } + } +} + +static int +pool_waitIdle(struct _WsbmBufStorage *buf, int lazy) +{ + struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); + + WSBM_MUTEX_LOCK(&buf->mutex); + waitIdleLocked(sBuf, lazy); + WSBM_MUTEX_UNLOCK(&buf->mutex); + + return 0; +} + +static int +pool_map(struct _WsbmBufStorage *buf, unsigned mode, void **virtual) +{ + struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); + + *virtual = sBuf->virtual; + + return 0; +} + +static void +pool_releaseFromCpu(struct _WsbmBufStorage *buf, unsigned mode) +{ + struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); + + if (wsbmAtomicDecZero(&sBuf->writers)) + WSBM_COND_BROADCAST(&sBuf->event); +} + +static int +pool_syncForCpu(struct _WsbmBufStorage *buf, unsigned mode) +{ + struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); + int ret = 0; + + WSBM_MUTEX_LOCK(&buf->mutex); + if ((mode & WSBM_SYNCCPU_DONT_BLOCK)) { + int signaled; + + if (sBuf->unFenced) { + ret = -EBUSY; + goto out_unlock; + } + + if (sBuf->isSlabBuffer) + signaled = (sBuf->fence == NULL) || + wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType); + else + signaled = (sBuf->fence == NULL) || + wsbmFenceSignaled(sBuf->fence, sBuf->fenceType); + + ret = 0; + if (signaled) { + wsbmFenceUnreference(&sBuf->fence); + wsbmAtomicInc(&sBuf->writers); + } else + ret = -EBUSY; + goto out_unlock; + } + waitIdleLocked(sBuf, 0); + wsbmAtomicInc(&sBuf->writers); + out_unlock: + WSBM_MUTEX_UNLOCK(&buf->mutex); + return ret; +} + +static void +pool_unmap(struct _WsbmBufStorage *buf) +{ + ; +} + +static unsigned long +pool_poolOffset(struct _WsbmBufStorage *buf) +{ + struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); + + return sBuf->start; +} + +static unsigned long +pool_size(struct _WsbmBufStorage *buf) +{ + struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); + + if (!sBuf->isSlabBuffer) + return sBuf->requestedSize; + + return sBuf->parent->header->bufSize; +} + +static struct _WsbmKernelBuf * +pool_kernel(struct _WsbmBufStorage *buf) +{ + struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); + + return (sBuf->isSlabBuffer) ? &sBuf->parent->kbo->kBuf : &sBuf->kBuf; +} + +static unsigned long +pool_offset(struct _WsbmBufStorage *buf) +{ + return pool_kernel(buf)->gpuOffset + pool_poolOffset(buf); +} + +static void +pool_fence(struct _WsbmBufStorage *buf, struct _WsbmFenceObject *fence) +{ + struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); + struct _WsbmKernelBuf *kBuf; + + WSBM_MUTEX_LOCK(&buf->mutex); + if (sBuf->fence) + wsbmFenceUnreference(&sBuf->fence); + + kBuf = pool_kernel(buf); + sBuf->fenceType = kBuf->fence_type_mask; + if (!wsbmFenceSignaledCached(fence, sBuf->fenceType)) + sBuf->fence = wsbmFenceReference(fence); + + sBuf->unFenced = 0; + WSBM_COND_BROADCAST(&sBuf->event); + WSBM_MUTEX_UNLOCK(&buf->mutex); +} + +static int +pool_validate(struct _WsbmBufStorage *buf, + uint64_t set_flags, uint64_t clr_flags) +{ + struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); + + WSBM_MUTEX_LOCK(&buf->mutex); + while (wsbmAtomicRead(&sBuf->writers) != 0) { + WSBM_COND_WAIT(&sBuf->event, &buf->mutex); + } + + sBuf->unFenced = 1; + WSBM_MUTEX_UNLOCK(&buf->mutex); + return 0; +} + +static void +pool_unvalidate(struct _WsbmBufStorage *buf) +{ + struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); + + WSBM_MUTEX_LOCK(&buf->mutex); + if (sBuf->unFenced) { + sBuf->unFenced = 0; + WSBM_COND_BROADCAST(&sBuf->event); + } + WSBM_MUTEX_UNLOCK(&buf->mutex); +} + +struct _WsbmSlabCache * +wsbmSlabCacheInit(uint32_t checkIntervalMsec, uint32_t slabTimeoutMsec) +{ + struct _WsbmSlabCache *tmp; + + tmp = calloc(1, sizeof(*tmp)); + if (!tmp) + return NULL; + + WSBM_MUTEX_INIT(&tmp->mutex); + WSBM_MUTEX_LOCK(&tmp->mutex); + tmp->slabTimeout.tv_usec = slabTimeoutMsec * 1000; + tmp->slabTimeout.tv_sec = tmp->slabTimeout.tv_usec / 1000000; + tmp->slabTimeout.tv_usec -= tmp->slabTimeout.tv_sec * 1000000; + + tmp->checkInterval.tv_usec = checkIntervalMsec * 1000; + tmp->checkInterval.tv_sec = tmp->checkInterval.tv_usec / 1000000; + tmp->checkInterval.tv_usec -= tmp->checkInterval.tv_sec * 1000000; + + gettimeofday(&tmp->nextCheck, NULL); + wsbmTimeAdd(&tmp->nextCheck, &tmp->checkInterval); + WSBMINITLISTHEAD(&tmp->timeoutList); + WSBMINITLISTHEAD(&tmp->unCached); + WSBMINITLISTHEAD(&tmp->cached); + WSBM_MUTEX_UNLOCK(&tmp->mutex); + + return tmp; +} + +void +wsbmSlabCacheFinish(struct _WsbmSlabCache *cache) +{ + struct timeval time; + + time = cache->nextCheck; + WSBM_MUTEX_LOCK(&cache->mutex); + wsbmTimeAdd(&time, &cache->checkInterval); + wsbmFreeTimeoutKBOsLocked(cache, &time); + WSBM_MUTEX_UNLOCK(&cache->mutex); + + assert(cache->timeoutList.next == &cache->timeoutList); + assert(cache->unCached.next == &cache->unCached); + assert(cache->cached.next == &cache->cached); + + WSBM_MUTEX_FREE(&cache->mutex); + free(cache); +} + +static void +wsbmInitSizeHeader(struct _WsbmSlabPool *slabPool, uint32_t size, + struct _WsbmSlabSizeHeader *header) +{ + WSBM_MUTEX_INIT(&header->mutex); + WSBM_MUTEX_LOCK(&header->mutex); + + WSBMINITLISTHEAD(&header->slabs); + WSBMINITLISTHEAD(&header->freeSlabs); + WSBMINITLISTHEAD(&header->delayedBuffers); + + header->numDelayed = 0; + header->slabPool = slabPool; + header->bufSize = size; + + WSBM_MUTEX_UNLOCK(&header->mutex); +} + +static void +wsbmFinishSizeHeader(struct _WsbmSlabSizeHeader *header) +{ + struct _WsbmListHead *list, *next; + struct _WsbmSlabBuffer *sBuf; + + WSBM_MUTEX_LOCK(&header->mutex); + WSBMLISTFOREACHSAFE(list, next, &header->delayedBuffers) { + sBuf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head); + + if (sBuf->fence) { + (void)wsbmFenceFinish(sBuf->fence, sBuf->fenceType, 0); + wsbmFenceUnreference(&sBuf->fence); + } + header->numDelayed--; + wsbmSlabFreeBufferLocked(sBuf); + } + WSBM_MUTEX_UNLOCK(&header->mutex); + WSBM_MUTEX_FREE(&header->mutex); +} + +static void +pool_takedown(struct _WsbmBufferPool *pool) +{ + struct _WsbmSlabPool *slabPool = slabPoolFromPool(pool); + int i; + + for (i = 0; i < slabPool->numBuckets; ++i) { + wsbmFinishSizeHeader(&slabPool->headers[i]); + } + + free(slabPool->headers); + free(slabPool->bucketSizes); + free(slabPool); +} + +struct _WsbmBufferPool * +wsbmSlabPoolInit(int fd, + uint32_t devOffset, + uint32_t placement, + uint32_t validMask, + uint32_t smallestSize, + uint32_t numSizes, + uint32_t desiredNumBuffers, + uint32_t maxSlabSize, + uint32_t pageAlignment, struct _WsbmSlabCache *cache) +{ + struct _WsbmBufferPool *pool; + struct _WsbmSlabPool *slabPool; + uint32_t i; + + slabPool = calloc(1, sizeof(*slabPool)); + if (!slabPool) + return NULL; + + pool = &slabPool->pool; + + slabPool->bucketSizes = calloc(numSizes, sizeof(*slabPool->bucketSizes)); + if (!slabPool->bucketSizes) + goto out_err0; + + slabPool->headers = calloc(numSizes, sizeof(*slabPool->headers)); + if (!slabPool->headers) + goto out_err1; + + slabPool->devOffset = devOffset; + slabPool->cache = cache; + slabPool->proposedPlacement = placement; + slabPool->validMask = validMask; + slabPool->numBuckets = numSizes; + slabPool->pageSize = getpagesize(); + slabPool->pageAlignment = pageAlignment; + slabPool->maxSlabSize = maxSlabSize; + slabPool->desiredNumBuffers = desiredNumBuffers; + + for (i = 0; i < slabPool->numBuckets; ++i) { + slabPool->bucketSizes[i] = (smallestSize << i); + wsbmInitSizeHeader(slabPool, slabPool->bucketSizes[i], + &slabPool->headers[i]); + } + + pool->fd = fd; + pool->map = &pool_map; + pool->unmap = &pool_unmap; + pool->destroy = &pool_destroy; + pool->offset = &pool_offset; + pool->poolOffset = &pool_poolOffset; + pool->size = &pool_size; + pool->create = &pool_create; + pool->fence = &pool_fence; + pool->kernel = &pool_kernel; + pool->validate = &pool_validate; + pool->unvalidate = &pool_unvalidate; + pool->waitIdle = &pool_waitIdle; + pool->takeDown = &pool_takedown; + pool->releasefromcpu = &pool_releaseFromCpu; + pool->syncforcpu = &pool_syncForCpu; + + return pool; + + out_err1: + free(slabPool->bucketSizes); + out_err0: + free(slabPool); + + return NULL; +} diff --git a/src/wsbm_ttmpool.c b/src/wsbm_ttmpool.c new file mode 100644 index 0000000..5867428 --- /dev/null +++ b/src/wsbm_ttmpool.c @@ -0,0 +1,509 @@ +/************************************************************************** + * + * Copyright 2006-2008 Tungsten Graphics, Inc., Cedar Park, Tx., USA + * All Rights Reserved. + * Copyright 2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellström + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include "wsbm_pool.h" +#include "assert.h" +#include "wsbm_priv.h" +#include "wsbm_manager.h" +#include "ttm/ttm_placement_user.h" + +#define DRMRESTARTCOMMANDWRITE(_fd, _val, _arg, _ret) \ + do { \ + (_ret) = drmCommandWrite(_fd, _val, &(_arg), sizeof(_arg)); \ + } while ((_ret) == -EAGAIN || (_ret) == -ERESTART); \ + +#define DRMRESTARTCOMMANDWRITEREAD(_fd, _val, _arg, _ret) \ + do { \ + (_ret) = drmCommandWriteRead(_fd, _val, &(_arg), sizeof(_arg)); \ + } while ((_ret) == -EAGAIN || (_ret) == -ERESTART); \ + +/* + * Buffer pool implementation using DRM buffer objects as wsbm buffer objects. + */ + +struct _TTMBuffer +{ + struct _WsbmBufStorage buf; + struct _WsbmCond event; + + /* + * Remains constant after creation. + */ + + uint64_t requestedSize; + uint64_t mapHandle; + uint64_t realSize; + + /* + * Protected by the kernel lock. + */ + + struct _WsbmKernelBuf kBuf; + + /* + * Protected by the mutex. + */ + + void *virtual; + int syncInProgress; + unsigned readers; + unsigned writers; +}; + +struct _TTMPool +{ + struct _WsbmBufferPool pool; + unsigned int pageSize; + unsigned int devOffset; +}; + +static inline struct _TTMPool * +ttmGetPool(struct _TTMBuffer *dBuf) +{ + return containerOf(dBuf->buf.pool, struct _TTMPool, pool); +} + +static inline struct _TTMBuffer * +ttmBuffer(struct _WsbmBufStorage *buf) +{ + return containerOf(buf, struct _TTMBuffer, buf); +} + +static struct _WsbmBufStorage * +pool_create(struct _WsbmBufferPool *pool, + unsigned long size, uint32_t placement, unsigned alignment) +{ + struct _TTMBuffer *dBuf = (struct _TTMBuffer *) + calloc(1, sizeof(*dBuf)); + struct _TTMPool *ttmPool = containerOf(pool, struct _TTMPool, pool); + int ret; + unsigned pageSize = ttmPool->pageSize; + union ttm_pl_create_arg arg; + + if (!dBuf) + return NULL; + + if ((alignment > pageSize) && (alignment % pageSize)) + goto out_err0; + + ret = wsbmBufStorageInit(&dBuf->buf, pool); + if (ret) + goto out_err0; + + ret = WSBM_COND_INIT(&dBuf->event); + if (ret) + goto out_err1; + + arg.req.size = size; + arg.req.placement = placement; + arg.req.page_alignment = alignment / pageSize; + + DRMRESTARTCOMMANDWRITEREAD(pool->fd, ttmPool->devOffset + TTM_PL_CREATE, + arg, ret); + + if (ret) + goto out_err2; + + dBuf->requestedSize = size; + dBuf->kBuf.gpuOffset = arg.rep.gpu_offset; + dBuf->mapHandle = arg.rep.map_handle; + dBuf->realSize = arg.rep.bo_size; + dBuf->kBuf.placement = arg.rep.placement; + dBuf->kBuf.handle = arg.rep.handle; + + return &dBuf->buf; + + out_err2: + WSBM_COND_FREE(&dBuf->event); + out_err1: + wsbmBufStorageTakedown(&dBuf->buf); + out_err0: + free(dBuf); + return NULL; +} + +static struct _WsbmBufStorage * +pool_reference(struct _WsbmBufferPool *pool, unsigned handle) +{ + struct _TTMBuffer *dBuf = (struct _TTMBuffer *)calloc(1, sizeof(*dBuf)); + struct _TTMPool *ttmPool = containerOf(pool, struct _TTMPool, pool); + union ttm_pl_reference_arg arg; + int ret; + + if (!dBuf) + return NULL; + + ret = wsbmBufStorageInit(&dBuf->buf, pool); + if (ret) + goto out_err0; + + ret = WSBM_COND_INIT(&dBuf->event); + if (ret) + goto out_err1; + + arg.req.handle = handle; + ret = drmCommandWriteRead(pool->fd, ttmPool->devOffset + TTM_PL_REFERENCE, + &arg, sizeof(arg)); + + if (ret) + goto out_err2; + + dBuf->requestedSize = arg.rep.bo_size; + dBuf->kBuf.gpuOffset = arg.rep.gpu_offset; + dBuf->mapHandle = arg.rep.map_handle; + dBuf->realSize = arg.rep.bo_size; + dBuf->kBuf.placement = arg.rep.placement; + dBuf->kBuf.handle = arg.rep.handle; + dBuf->kBuf.fence_type_mask = arg.rep.sync_object_arg; + + return &dBuf->buf; + + out_err2: + WSBM_COND_FREE(&dBuf->event); + out_err1: + wsbmBufStorageTakedown(&dBuf->buf); + out_err0: + free(dBuf); + return NULL; +} + +static void +pool_destroy(struct _WsbmBufStorage **buf) +{ + struct _TTMBuffer *dBuf = ttmBuffer(*buf); + struct _TTMPool *ttmPool = ttmGetPool(dBuf); + struct ttm_pl_reference_req arg; + + *buf = NULL; + if (dBuf->virtual != NULL) { + (void)munmap(dBuf->virtual, dBuf->requestedSize); + dBuf->virtual = NULL; + } + arg.handle = dBuf->kBuf.handle; + (void)drmCommandWrite(dBuf->buf.pool->fd, + ttmPool->devOffset + TTM_PL_UNREF, + &arg, sizeof(arg)); + + WSBM_COND_FREE(&dBuf->event); + wsbmBufStorageTakedown(&dBuf->buf); + free(dBuf); +} + +static int +syncforcpu_locked(struct _WsbmBufStorage *buf, unsigned mode) +{ + uint32_t kmode = 0; + struct _TTMBuffer *dBuf = ttmBuffer(buf); + struct _TTMPool *ttmPool = ttmGetPool(dBuf); + unsigned int readers; + unsigned int writers; + int ret = 0; + + while (dBuf->syncInProgress) + WSBM_COND_WAIT(&dBuf->event, &buf->mutex); + + readers = dBuf->readers; + writers = dBuf->writers; + + if ((mode & WSBM_SYNCCPU_READ) && (++dBuf->readers == 1)) + kmode |= TTM_PL_SYNCCPU_MODE_READ; + + if ((mode & WSBM_SYNCCPU_WRITE) && (++dBuf->writers == 1)) + kmode |= TTM_PL_SYNCCPU_MODE_WRITE; + + if (kmode) { + struct ttm_pl_synccpu_arg arg; + + if (mode & WSBM_SYNCCPU_DONT_BLOCK) + kmode |= TTM_PL_SYNCCPU_MODE_NO_BLOCK; + + dBuf->syncInProgress = 1; + + /* + * This might be a lengthy wait, so + * release the mutex. + */ + + WSBM_MUTEX_UNLOCK(&buf->mutex); + + arg.handle = dBuf->kBuf.handle; + arg.access_mode = kmode; + arg.op = TTM_PL_SYNCCPU_OP_GRAB; + + DRMRESTARTCOMMANDWRITE(dBuf->buf.pool->fd, + ttmPool->devOffset + TTM_PL_SYNCCPU, arg, ret); + + WSBM_MUTEX_LOCK(&buf->mutex); + dBuf->syncInProgress = 0; + WSBM_COND_BROADCAST(&dBuf->event); + + if (ret) { + dBuf->readers = readers; + dBuf->writers = writers; + } + } + + return ret; +} + +static int +releasefromcpu_locked(struct _WsbmBufStorage *buf, unsigned mode) +{ + uint32_t kmode = 0; + struct _TTMBuffer *dBuf = ttmBuffer(buf); + struct _TTMPool *ttmPool = ttmGetPool(dBuf); + int ret = 0; + + while (dBuf->syncInProgress) + WSBM_COND_WAIT(&dBuf->event, &buf->mutex); + + if ((mode & WSBM_SYNCCPU_READ) && (--dBuf->readers == 0)) + kmode |= TTM_PL_SYNCCPU_MODE_READ; + + if ((mode & WSBM_SYNCCPU_WRITE) && (--dBuf->writers == 0)) + kmode |= TTM_PL_SYNCCPU_MODE_WRITE; + + if (kmode) { + struct ttm_pl_synccpu_arg arg; + + arg.handle = dBuf->kBuf.handle; + arg.access_mode = kmode; + arg.op = TTM_PL_SYNCCPU_OP_RELEASE; + + DRMRESTARTCOMMANDWRITE(dBuf->buf.pool->fd, + ttmPool->devOffset + TTM_PL_SYNCCPU, arg, ret); + + } + + return ret; +} + +static int +pool_syncforcpu(struct _WsbmBufStorage *buf, unsigned mode) +{ + int ret; + + WSBM_MUTEX_LOCK(&buf->mutex); + ret = syncforcpu_locked(buf, mode); + WSBM_MUTEX_UNLOCK(&buf->mutex); + return ret; +} + +static void +pool_releasefromcpu(struct _WsbmBufStorage *buf, unsigned mode) +{ + WSBM_MUTEX_LOCK(&buf->mutex); + (void)releasefromcpu_locked(buf, mode); + WSBM_MUTEX_UNLOCK(&buf->mutex); +} + +static int +pool_map(struct _WsbmBufStorage *buf, unsigned mode, void **virtual) +{ + struct _TTMBuffer *dBuf = ttmBuffer(buf); + void *virt; + int ret = 0; + + WSBM_MUTEX_LOCK(&buf->mutex); + + /* + * mmaps are expensive, so we only really unmap if + * we destroy the buffer. + */ + + if (dBuf->virtual == NULL) { + virt = mmap(0, dBuf->requestedSize, + PROT_READ | PROT_WRITE, MAP_SHARED, + buf->pool->fd, dBuf->mapHandle); + if (virt == MAP_FAILED) { + ret = -errno; + goto out_unlock; + } + dBuf->virtual = virt; + } + + *virtual = dBuf->virtual; + out_unlock: + + WSBM_MUTEX_UNLOCK(&buf->mutex); + + return ret; +} + +static void +pool_unmap(struct _WsbmBufStorage *buf) +{ + ; +} + +static unsigned long +pool_offset(struct _WsbmBufStorage *buf) +{ + struct _TTMBuffer *dBuf = ttmBuffer(buf); + + return dBuf->kBuf.gpuOffset; +} + +static unsigned long +pool_poolOffset(struct _WsbmBufStorage *buf) +{ + return 0; +} + +static uint32_t +pool_placement(struct _WsbmBufStorage *buf) +{ + struct _TTMBuffer *dBuf = ttmBuffer(buf); + + return dBuf->kBuf.placement; +} + +static unsigned long +pool_size(struct _WsbmBufStorage *buf) +{ + struct _TTMBuffer *dBuf = ttmBuffer(buf); + + return dBuf->realSize; +} + +static void +pool_fence(struct _WsbmBufStorage *buf, struct _WsbmFenceObject *fence) +{ + /* + * Noop. The kernel handles all fencing. + */ +} + +static int +pool_waitIdle(struct _WsbmBufStorage *buf, int lazy) +{ + struct _TTMBuffer *dBuf = ttmBuffer(buf); + struct _TTMPool *ttmPool = ttmGetPool(dBuf); + struct ttm_pl_waitidle_arg req; + struct _WsbmBufferPool *pool = buf->pool; + int ret; + + req.handle = dBuf->kBuf.handle; + req.mode = (lazy) ? TTM_PL_WAITIDLE_MODE_LAZY : 0; + + DRMRESTARTCOMMANDWRITE(pool->fd, ttmPool->devOffset + TTM_PL_WAITIDLE, + req, ret); + + return ret; +} + +static void +pool_takedown(struct _WsbmBufferPool *pool) +{ + struct _TTMPool *ttmPool = containerOf(pool, struct _TTMPool, pool); + + free(ttmPool); +} + +static int +pool_setStatus(struct _WsbmBufStorage *buf, uint32_t set_placement, + uint32_t clr_placement) +{ + struct _TTMBuffer *dBuf = ttmBuffer(buf); + struct _TTMPool *ttmPool = ttmGetPool(dBuf); + union ttm_pl_setstatus_arg arg; + struct ttm_pl_setstatus_req *req = &arg.req; + struct ttm_pl_rep *rep = &arg.rep; + struct _WsbmBufferPool *pool = buf->pool; + int ret; + + req->handle = dBuf->kBuf.handle; + req->set_placement = set_placement; + req->clr_placement = clr_placement; + + DRMRESTARTCOMMANDWRITEREAD(pool->fd, + ttmPool->devOffset + TTM_PL_SETSTATUS, + arg, ret); + + if (!ret) { + dBuf->kBuf.gpuOffset = rep->gpu_offset; + dBuf->kBuf.placement = rep->placement; + } + + return ret; +} + +static struct _WsbmKernelBuf * +pool_kernel(struct _WsbmBufStorage *buf) +{ + return (void *)&ttmBuffer(buf)->kBuf; +} + +struct _WsbmBufferPool * +wsbmTTMPoolInit(int fd, unsigned int devOffset) +{ + struct _TTMPool *ttmPool; + struct _WsbmBufferPool *pool; + + ttmPool = (struct _TTMPool *)calloc(1, sizeof(*ttmPool)); + + if (!ttmPool) + return NULL; + + ttmPool->pageSize = getpagesize(); + ttmPool->devOffset = devOffset; + pool = &ttmPool->pool; + + pool->fd = fd; + pool->map = &pool_map; + pool->unmap = &pool_unmap; + pool->syncforcpu = &pool_syncforcpu; + pool->releasefromcpu = &pool_releasefromcpu; + pool->destroy = &pool_destroy; + pool->offset = &pool_offset; + pool->poolOffset = &pool_poolOffset; + pool->placement = &pool_placement; + pool->size = &pool_size; + pool->create = &pool_create; + pool->fence = &pool_fence; + pool->kernel = &pool_kernel; + pool->validate = NULL; + pool->unvalidate = NULL; + pool->waitIdle = &pool_waitIdle; + pool->takeDown = &pool_takedown; + pool->createByReference = &pool_reference; + pool->setStatus = &pool_setStatus; + return pool; +} diff --git a/src/wsbm_userpool.c b/src/wsbm_userpool.c new file mode 100644 index 0000000..f7833b7 --- /dev/null +++ b/src/wsbm_userpool.c @@ -0,0 +1,689 @@ +/************************************************************************** + * + * Copyright 2008 Tungsten Graphics, Inc., Cedar Park, TX., USA + * All Rights Reserved. + * Copyright 2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include "wsbm_pool.h" +#include "wsbm_fencemgr.h" +#include "wsbm_manager.h" +#include "wsbm_mm.h" +#include "wsbm_priv.h" + +/* + * Malloced memory must be aligned to 16 bytes, since that's what + * the DMA bitblt requires. + */ + +#define WSBM_USER_ALIGN_ADD 16 +#define WSBM_USER_ALIGN_SYSMEM(_val) \ + ((void *)(((unsigned long) (_val) + 15) & ~15)) + +struct _WsbmUserBuffer +{ + struct _WsbmBufStorage buf; + struct _WsbmKernelBuf kBuf; + + /* Protected by the pool mutex */ + + struct _WsbmListHead lru; + struct _WsbmListHead delayed; + + /* Protected by the buffer mutex */ + + unsigned long size; + unsigned long alignment; + + struct _WsbmCond event; + uint32_t proposedPlacement; + uint32_t newFenceType; + + void *map; + void *sysmem; + int unFenced; + struct _WsbmFenceObject *fence; + struct _WsbmMMNode *node; + + struct _WsbmAtomic writers; +}; + +struct _WsbmUserPool +{ + /* + * Constant after initialization. + */ + + struct _WsbmBufferPool pool; + unsigned long agpOffset; + unsigned long agpMap; + unsigned long agpSize; + unsigned long vramOffset; + unsigned long vramMap; + unsigned long vramSize; + struct _WsbmMutex mutex; + struct _WsbmListHead delayed; + struct _WsbmListHead vramLRU; + struct _WsbmListHead agpLRU; + struct _WsbmMM vramMM; + struct _WsbmMM agpMM; + uint32_t(*fenceTypes) (uint64_t); +}; + +static inline struct _WsbmUserPool * +userPool(struct _WsbmUserBuffer *buf) +{ + return containerOf(buf->buf.pool, struct _WsbmUserPool, pool); +} + +static inline struct _WsbmUserBuffer * +userBuf(struct _WsbmBufStorage *buf) +{ + return containerOf(buf, struct _WsbmUserBuffer, buf); +} + +static void +waitIdleLocked(struct _WsbmBufStorage *buf, int lazy) +{ + struct _WsbmUserBuffer *vBuf = userBuf(buf); + + while (vBuf->unFenced || vBuf->fence != NULL) { + if (vBuf->unFenced) + WSBM_COND_WAIT(&vBuf->event, &buf->mutex); + + if (vBuf->fence != NULL) { + if (!wsbmFenceSignaled(vBuf->fence, vBuf->kBuf.fence_type_mask)) { + struct _WsbmFenceObject *fence = + wsbmFenceReference(vBuf->fence); + + WSBM_MUTEX_UNLOCK(&buf->mutex); + (void)wsbmFenceFinish(fence, vBuf->kBuf.fence_type_mask, + lazy); + WSBM_MUTEX_LOCK(&buf->mutex); + + if (vBuf->fence == fence) + wsbmFenceUnreference(&vBuf->fence); + + wsbmFenceUnreference(&fence); + } else { + wsbmFenceUnreference(&vBuf->fence); + } + } + } +} + +static int +pool_waitIdle(struct _WsbmBufStorage *buf, int lazy) +{ + WSBM_MUTEX_UNLOCK(&buf->mutex); + waitIdleLocked(buf, lazy); + WSBM_MUTEX_UNLOCK(&buf->mutex); + + return 0; +} + +static int +evict_lru(struct _WsbmListHead *lru) +{ + struct _WsbmUserBuffer *vBuf; + struct _WsbmUserPool *p; + struct _WsbmListHead *list = lru->next; + int err; + + if (list == lru) { + return -ENOMEM; + } + + vBuf = WSBMLISTENTRY(list, struct _WsbmUserBuffer, lru); + p = userPool(vBuf); + WSBM_MUTEX_UNLOCK(&p->mutex); + WSBM_MUTEX_LOCK(&vBuf->buf.mutex); + WSBM_MUTEX_LOCK(&p->mutex); + + vBuf->sysmem = malloc(vBuf->size + WSBM_USER_ALIGN_ADD); + + if (!vBuf->sysmem) { + err = -ENOMEM; + goto out_unlock; + } + + (void)wsbmFenceFinish(vBuf->fence, vBuf->kBuf.fence_type_mask, 0); + wsbmFenceUnreference(&vBuf->fence); + + memcpy(WSBM_USER_ALIGN_SYSMEM(vBuf->sysmem), vBuf->map, vBuf->size); + WSBMLISTDELINIT(&vBuf->lru); + vBuf->kBuf.placement = WSBM_PL_FLAG_SYSTEM; + vBuf->map = WSBM_USER_ALIGN_SYSMEM(vBuf->sysmem); + + /* + * FIXME: Free memory. + */ + + err = 0; + out_unlock: + WSBM_MUTEX_UNLOCK(&vBuf->buf.mutex); + return err; +} + +static struct _WsbmBufStorage * +pool_create(struct _WsbmBufferPool *pool, + unsigned long size, uint32_t placement, unsigned alignment) +{ + struct _WsbmUserPool *p = containerOf(pool, struct _WsbmUserPool, pool); + struct _WsbmUserBuffer *vBuf = calloc(1, sizeof(*vBuf)); + + if (!vBuf) + return NULL; + + wsbmBufStorageInit(&vBuf->buf, pool); + vBuf->sysmem = NULL; + vBuf->proposedPlacement = placement; + vBuf->size = size; + vBuf->alignment = alignment; + + WSBMINITLISTHEAD(&vBuf->lru); + WSBMINITLISTHEAD(&vBuf->delayed); + WSBM_MUTEX_LOCK(&p->mutex); + + if (placement & WSBM_PL_FLAG_TT) { + vBuf->node = wsbmMMSearchFree(&p->agpMM, size, alignment, 1); + if (vBuf->node) + vBuf->node = wsbmMMGetBlock(vBuf->node, size, alignment); + + if (vBuf->node) { + vBuf->kBuf.placement = WSBM_PL_FLAG_TT; + vBuf->kBuf.gpuOffset = p->agpOffset + vBuf->node->start; + vBuf->map = (void *)(p->agpMap + vBuf->node->start); + WSBMLISTADDTAIL(&vBuf->lru, &p->agpLRU); + goto have_mem; + } + } + + if (placement & WSBM_PL_FLAG_VRAM) { + vBuf->node = wsbmMMSearchFree(&p->vramMM, size, alignment, 1); + if (vBuf->node) + vBuf->node = wsbmMMGetBlock(vBuf->node, size, alignment); + + if (vBuf->node) { + vBuf->kBuf.placement = WSBM_PL_FLAG_VRAM; + vBuf->kBuf.gpuOffset = p->vramOffset + vBuf->node->start; + vBuf->map = (void *)(p->vramMap + vBuf->node->start); + WSBMLISTADDTAIL(&vBuf->lru, &p->vramLRU); + goto have_mem; + } + } + + if ((placement & WSBM_PL_FLAG_NO_EVICT) + && !(placement & WSBM_PL_FLAG_SYSTEM)) { + WSBM_MUTEX_UNLOCK(&p->mutex); + goto out_err; + } + + vBuf->sysmem = malloc(size + WSBM_USER_ALIGN_ADD); + vBuf->kBuf.placement = WSBM_PL_FLAG_SYSTEM; + vBuf->map = WSBM_USER_ALIGN_SYSMEM(vBuf->sysmem); + + have_mem: + + WSBM_MUTEX_UNLOCK(&p->mutex); + if (vBuf->sysmem != NULL + || (!(vBuf->kBuf.placement & WSBM_PL_FLAG_SYSTEM))) + return &vBuf->buf; + out_err: + free(vBuf); + return NULL; +} + +static int +pool_validate(struct _WsbmBufStorage *buf, uint64_t set_flags, + uint64_t clr_flags) +{ + struct _WsbmUserBuffer *vBuf = userBuf(buf); + struct _WsbmUserPool *p = userPool(vBuf); + int err = -ENOMEM; + + WSBM_MUTEX_LOCK(&buf->mutex); + + while (wsbmAtomicRead(&vBuf->writers) != 0) + WSBM_COND_WAIT(&vBuf->event, &buf->mutex); + + vBuf->unFenced = 1; + + WSBM_MUTEX_LOCK(&p->mutex); + WSBMLISTDELINIT(&vBuf->lru); + + vBuf->proposedPlacement = + (vBuf->proposedPlacement | set_flags) & ~clr_flags; + + if ((vBuf->proposedPlacement & vBuf->kBuf.placement & WSBM_PL_MASK_MEM) == + vBuf->kBuf.placement) { + err = 0; + goto have_mem; + } + + /* + * We're moving to another memory region, so evict first and we'll + * do a sw copy to the other region. + */ + + if (!(vBuf->kBuf.placement & WSBM_PL_FLAG_SYSTEM)) { + struct _WsbmListHead tmpLRU; + + WSBMINITLISTHEAD(&tmpLRU); + WSBMLISTADDTAIL(&tmpLRU, &vBuf->lru); + err = evict_lru(&tmpLRU); + if (err) + goto have_mem; + } + + if (vBuf->proposedPlacement & WSBM_PL_FLAG_TT) { + do { + vBuf->node = + wsbmMMSearchFree(&p->agpMM, vBuf->size, vBuf->alignment, 1); + if (vBuf->node) + vBuf->node = + wsbmMMGetBlock(vBuf->node, vBuf->size, vBuf->alignment); + + if (vBuf->node) { + vBuf->kBuf.placement = WSBM_PL_FLAG_TT; + vBuf->kBuf.gpuOffset = p->agpOffset + vBuf->node->start; + vBuf->map = (void *)(p->agpMap + vBuf->node->start); + memcpy(vBuf->map, WSBM_USER_ALIGN_SYSMEM(vBuf->sysmem), + vBuf->size); + free(vBuf->sysmem); + goto have_mem; + } + } while (evict_lru(&p->agpLRU) == 0); + } + + if (vBuf->proposedPlacement & WSBM_PL_FLAG_VRAM) { + do { + vBuf->node = + wsbmMMSearchFree(&p->vramMM, vBuf->size, vBuf->alignment, 1); + if (vBuf->node) + vBuf->node = + wsbmMMGetBlock(vBuf->node, vBuf->size, vBuf->alignment); + + if (!err) { + vBuf->kBuf.placement = WSBM_PL_FLAG_VRAM; + vBuf->kBuf.gpuOffset = p->vramOffset + vBuf->node->start; + vBuf->map = (void *)(p->vramMap + vBuf->node->start); + memcpy(vBuf->map, WSBM_USER_ALIGN_SYSMEM(vBuf->sysmem), + vBuf->size); + free(vBuf->sysmem); + goto have_mem; + } + } while (evict_lru(&p->vramLRU) == 0); + } + + if (vBuf->proposedPlacement & WSBM_PL_FLAG_SYSTEM) + goto have_mem; + + err = -ENOMEM; + + have_mem: + vBuf->newFenceType = p->fenceTypes(set_flags); + WSBM_MUTEX_UNLOCK(&p->mutex); + WSBM_MUTEX_UNLOCK(&buf->mutex); + return err; +} + +static int +pool_setStatus(struct _WsbmBufStorage *buf, + uint32_t set_placement, uint32_t clr_placement) +{ + struct _WsbmUserBuffer *vBuf = userBuf(buf); + int ret; + + ret = pool_validate(buf, set_placement, clr_placement); + vBuf->unFenced = 0; + return ret; +} + +void +release_delayed_buffers(struct _WsbmUserPool *p) +{ + struct _WsbmUserBuffer *vBuf; + struct _WsbmListHead *list, *next; + + WSBM_MUTEX_LOCK(&p->mutex); + + /* + * We don't need to take the buffer mutexes in this loop, since + * the only other user is the evict_lru function, which has the + * pool mutex held when accessing the buffer fence member. + */ + + WSBMLISTFOREACHSAFE(list, next, &p->delayed) { + vBuf = WSBMLISTENTRY(list, struct _WsbmUserBuffer, delayed); + + if (!vBuf->fence + || wsbmFenceSignaled(vBuf->fence, vBuf->kBuf.fence_type_mask)) { + if (vBuf->fence) + wsbmFenceUnreference(&vBuf->fence); + + WSBMLISTDEL(&vBuf->delayed); + WSBMLISTDEL(&vBuf->lru); + + if ((vBuf->kBuf.placement & WSBM_PL_FLAG_SYSTEM) == 0) + wsbmMMPutBlock(vBuf->node); + else + free(vBuf->sysmem); + + free(vBuf); + } else + break; + + } + WSBM_MUTEX_UNLOCK(&p->mutex); +} + +static void +pool_destroy(struct _WsbmBufStorage **buf) +{ + struct _WsbmUserBuffer *vBuf = userBuf(*buf); + struct _WsbmUserPool *p = userPool(vBuf); + + *buf = NULL; + + WSBM_MUTEX_LOCK(&vBuf->buf.mutex); + if ((vBuf->fence + && !wsbmFenceSignaled(vBuf->fence, vBuf->kBuf.fence_type_mask))) { + WSBM_MUTEX_LOCK(&p->mutex); + WSBMLISTADDTAIL(&vBuf->delayed, &p->delayed); + WSBM_MUTEX_UNLOCK(&p->mutex); + WSBM_MUTEX_UNLOCK(&vBuf->buf.mutex); + return; + } + + if (vBuf->fence) + wsbmFenceUnreference(&vBuf->fence); + + WSBM_MUTEX_LOCK(&p->mutex); + WSBMLISTDEL(&vBuf->lru); + WSBM_MUTEX_UNLOCK(&p->mutex); + + if (!(vBuf->kBuf.placement & WSBM_PL_FLAG_SYSTEM)) + wsbmMMPutBlock(vBuf->node); + else + free(vBuf->sysmem); + + free(vBuf); + return; +} + +static int +pool_map(struct _WsbmBufStorage *buf, unsigned mode, void **virtual) +{ + struct _WsbmUserBuffer *vBuf = userBuf(buf); + + *virtual = vBuf->map; + return 0; +} + +static void +pool_unmap(struct _WsbmBufStorage *buf) +{ + ; +} + +static void +pool_releaseFromCpu(struct _WsbmBufStorage *buf, unsigned mode) +{ + struct _WsbmUserBuffer *vBuf = userBuf(buf); + + if (wsbmAtomicDecZero(&vBuf->writers)) + WSBM_COND_BROADCAST(&vBuf->event); + +} + +static int +pool_syncForCpu(struct _WsbmBufStorage *buf, unsigned mode) +{ + struct _WsbmUserBuffer *vBuf = userBuf(buf); + int ret = 0; + + WSBM_MUTEX_LOCK(&buf->mutex); + if ((mode & WSBM_SYNCCPU_DONT_BLOCK)) { + + if (vBuf->unFenced) { + ret = -EBUSY; + goto out_unlock; + } + + ret = 0; + if ((vBuf->fence == NULL) || + wsbmFenceSignaled(vBuf->fence, vBuf->kBuf.fence_type_mask)) { + wsbmFenceUnreference(&vBuf->fence); + wsbmAtomicInc(&vBuf->writers); + } else + ret = -EBUSY; + + goto out_unlock; + } + waitIdleLocked(buf, 0); + wsbmAtomicInc(&vBuf->writers); + out_unlock: + WSBM_MUTEX_UNLOCK(&buf->mutex); + return ret; +} + +static unsigned long +pool_offset(struct _WsbmBufStorage *buf) +{ + return userBuf(buf)->kBuf.gpuOffset; +} + +static unsigned long +pool_poolOffset(struct _WsbmBufStorage *buf) +{ + return 0UL; +} + +static unsigned long +pool_size(struct _WsbmBufStorage *buf) +{ + return userBuf(buf)->size; +} + +static void +pool_fence(struct _WsbmBufStorage *buf, struct _WsbmFenceObject *fence) +{ + struct _WsbmUserBuffer *vBuf = userBuf(buf); + struct _WsbmUserPool *p = userPool(vBuf); + + WSBM_MUTEX_LOCK(&buf->mutex); + + if (vBuf->fence) + wsbmFenceUnreference(&vBuf->fence); + + vBuf->fence = wsbmFenceReference(fence); + vBuf->unFenced = 0; + vBuf->kBuf.fence_type_mask = vBuf->newFenceType; + + WSBM_COND_BROADCAST(&vBuf->event); + WSBM_MUTEX_LOCK(&p->mutex); + if (vBuf->kBuf.placement & WSBM_PL_FLAG_VRAM) + WSBMLISTADDTAIL(&vBuf->lru, &p->vramLRU); + else if (vBuf->kBuf.placement & WSBM_PL_FLAG_TT) + WSBMLISTADDTAIL(&vBuf->lru, &p->agpLRU); + WSBM_MUTEX_UNLOCK(&p->mutex); + WSBM_MUTEX_UNLOCK(&buf->mutex); +} + +static void +pool_unvalidate(struct _WsbmBufStorage *buf) +{ + struct _WsbmUserBuffer *vBuf = userBuf(buf); + struct _WsbmUserPool *p = userPool(vBuf); + + WSBM_MUTEX_LOCK(&buf->mutex); + + if (!vBuf->unFenced) + goto out_unlock; + + vBuf->unFenced = 0; + WSBM_COND_BROADCAST(&vBuf->event); + WSBM_MUTEX_LOCK(&p->mutex); + if (vBuf->kBuf.placement & WSBM_PL_FLAG_VRAM) + WSBMLISTADDTAIL(&vBuf->lru, &p->vramLRU); + else if (vBuf->kBuf.placement & WSBM_PL_FLAG_TT) + WSBMLISTADDTAIL(&vBuf->lru, &p->agpLRU); + WSBM_MUTEX_UNLOCK(&p->mutex); + + out_unlock: + + WSBM_MUTEX_UNLOCK(&buf->mutex); +} + +static struct _WsbmKernelBuf * +pool_kernel(struct _WsbmBufStorage *buf) +{ + struct _WsbmUserBuffer *vBuf = userBuf(buf); + + return &vBuf->kBuf; +} + +static void +pool_takedown(struct _WsbmBufferPool *pool) +{ + struct _WsbmUserPool *p = containerOf(pool, struct _WsbmUserPool, pool); + int empty; + + do { + release_delayed_buffers(p); + WSBM_MUTEX_LOCK(&p->mutex); + empty = (p->delayed.next == &p->delayed); + WSBM_MUTEX_UNLOCK(&p->mutex); + + if (!empty) + usleep(1000); + + } while (!empty); + WSBM_MUTEX_LOCK(&p->mutex); + + while (evict_lru(&p->vramLRU) == 0) ; + while (evict_lru(&p->agpLRU) == 0) ; + + WSBM_MUTEX_UNLOCK(&p->mutex); + + wsbmMMtakedown(&p->agpMM); + wsbmMMtakedown(&p->vramMM); + + free(p); +} + +void +wsbmUserPoolClean(struct _WsbmBufferPool *pool, int cleanVram, int cleanAgp) +{ + struct _WsbmUserPool *p = containerOf(pool, struct _WsbmUserPool, pool); + + WSBM_MUTEX_LOCK(&p->mutex); + if (cleanVram) + while (evict_lru(&p->vramLRU) == 0) ; + if (cleanAgp) + while (evict_lru(&p->agpLRU) == 0) ; + WSBM_MUTEX_UNLOCK(&p->mutex); +} + +struct _WsbmBufferPool * +wsbmUserPoolInit(void *vramAddr, + unsigned long vramStart, unsigned long vramSize, + void *agpAddr, unsigned long agpStart, + unsigned long agpSize, + uint32_t(*fenceTypes) (uint64_t set_flags)) +{ + struct _WsbmBufferPool *pool; + struct _WsbmUserPool *uPool; + int ret; + + uPool = calloc(1, sizeof(*uPool)); + if (!uPool) + goto out_err0; + + ret = WSBM_MUTEX_INIT(&uPool->mutex); + if (ret) + goto out_err0; + + ret = wsbmMMinit(&uPool->vramMM, 0, vramSize); + if (ret) + goto out_err1; + + ret = wsbmMMinit(&uPool->agpMM, 0, agpSize); + if (ret) + goto out_err2; + + WSBMINITLISTHEAD(&uPool->delayed); + WSBMINITLISTHEAD(&uPool->vramLRU); + WSBMINITLISTHEAD(&uPool->agpLRU); + + uPool->agpOffset = agpStart; + uPool->agpMap = (unsigned long)agpAddr; + uPool->vramOffset = vramStart; + uPool->vramMap = (unsigned long)vramAddr; + uPool->fenceTypes = fenceTypes; + + pool = &uPool->pool; + pool->map = &pool_map; + pool->unmap = &pool_unmap; + pool->destroy = &pool_destroy; + pool->offset = &pool_offset; + pool->poolOffset = &pool_poolOffset; + pool->size = &pool_size; + pool->create = &pool_create; + pool->fence = &pool_fence; + pool->unvalidate = &pool_unvalidate; + pool->kernel = &pool_kernel; + pool->validate = &pool_validate; + pool->waitIdle = &pool_waitIdle; + pool->takeDown = &pool_takedown; + pool->setStatus = &pool_setStatus; + pool->syncforcpu = &pool_syncForCpu; + pool->releasefromcpu = &pool_releaseFromCpu; + + return pool; + + out_err2: + wsbmMMtakedown(&uPool->vramMM); + out_err1: + WSBM_MUTEX_FREE(&uPool->mutex); + out_err0: + free(uPool); + + return NULL; +} diff --git a/src/wsbm_util.h b/src/wsbm_util.h new file mode 100644 index 0000000..13212c1 --- /dev/null +++ b/src/wsbm_util.h @@ -0,0 +1,76 @@ +/* + * This file is not copyrighted. + */ + +#ifndef _WSBM_UTIL_H_ +#define _WSBM_UTIL_H_ + +#include + +#ifndef containerOf +#define containerOf(__item, __type, __field) \ + ((__type *)(((char *) (__item)) - offsetof(__type, __field))) +#endif + +struct _WsbmListHead +{ + struct _WsbmListHead *prev; + struct _WsbmListHead *next; +}; + +#define WSBMINITLISTHEAD(__item) \ + do{ \ + (__item)->prev = (__item); \ + (__item)->next = (__item); \ + } while (0) + +#define WSBMLISTADD(__item, __list) \ + do { \ + (__item)->prev = (__list); \ + (__item)->next = (__list)->next; \ + (__list)->next->prev = (__item); \ + (__list)->next = (__item); \ + } while (0) + +#define WSBMLISTADDTAIL(__item, __list) \ + do { \ + (__item)->next = (__list); \ + (__item)->prev = (__list)->prev; \ + (__list)->prev->next = (__item); \ + (__list)->prev = (__item); \ + } while(0) + +#define WSBMLISTDEL(__item) \ + do { \ + (__item)->prev->next = (__item)->next; \ + (__item)->next->prev = (__item)->prev; \ + } while(0) + +#define WSBMLISTDELINIT(__item) \ + do { \ + (__item)->prev->next = (__item)->next; \ + (__item)->next->prev = (__item)->prev; \ + (__item)->next = (__item); \ + (__item)->prev = (__item); \ + } while(0) + +#define WSBMLISTFOREACH(__item, __list) \ + for((__item) = (__list)->next; (__item) != (__list); (__item) = (__item)->next) + +#define WSBMLISTFOREACHPREV(__item, __list) \ + for((__item) = (__list)->prev; (__item) != (__list); (__item) = (__item)->prev) + +#define WSBMLISTFOREACHSAFE(__item, __next, __list) \ + for((__item) = (__list)->next, (__next) = (__item)->next; \ + (__item) != (__list); \ + (__item) = (__next), (__next) = (__item)->next) + +#define WSBMLISTFOREACHPREVSAFE(__item, __prev, __list) \ + for((__item) = (__list)->prev, (__prev) = (__item->prev); \ + (__item) != (__list); \ + (__item) = (__prev), (__prev) = (__item)->prev) + +#define WSBMLISTENTRY(__item, __type, __field) \ + containerOf(__item, __type, __field) + +#endif -- 2.7.4