#include <drm/drm_legacy.h> /* for struct drm_dma_handle */
#include <drm/drm_gem.h>
#include <drm/drm_auth.h>
+#include <drm/drm_cache.h>
#include "i915_params.h"
#include "i915_reg.h"
+#include "i915_utils.h"
#include "intel_bios.h"
#include "intel_dpll_mgr.h"
-#include "intel_guc.h"
+#include "intel_uc.h"
#include "intel_lrc.h"
#include "intel_ringbuffer.h"
#include "i915_gem.h"
+#include "i915_gem_context.h"
#include "i915_gem_fence_reg.h"
#include "i915_gem_object.h"
#include "i915_gem_gtt.h"
#define DRIVER_NAME "i915"
#define DRIVER_DESC "Intel Graphics"
-#define DRIVER_DATE "20161121"
-#define DRIVER_TIMESTAMP 1479717903
+#define DRIVER_DATE "20170306"
+#define DRIVER_TIMESTAMP 1488785683
#undef WARN_ON
/* Many gcc seem to no see through this and fall over :( */
#define i915_inject_load_failure() \
__i915_inject_load_failure(__func__, __LINE__)
+typedef struct {
+ uint32_t val;
+} uint_fixed_16_16_t;
+
+#define FP_16_16_MAX ({ \
+ uint_fixed_16_16_t fp; \
+ fp.val = UINT_MAX; \
+ fp; \
+})
+
+static inline uint_fixed_16_16_t u32_to_fixed_16_16(uint32_t val)
+{
+ uint_fixed_16_16_t fp;
+
+ WARN_ON(val >> 16);
+
+ fp.val = val << 16;
+ return fp;
+}
+
+static inline uint32_t fixed_16_16_to_u32_round_up(uint_fixed_16_16_t fp)
+{
+ return DIV_ROUND_UP(fp.val, 1 << 16);
+}
+
+static inline uint32_t fixed_16_16_to_u32(uint_fixed_16_16_t fp)
+{
+ return fp.val >> 16;
+}
+
+static inline uint_fixed_16_16_t min_fixed_16_16(uint_fixed_16_16_t min1,
+ uint_fixed_16_16_t min2)
+{
+ uint_fixed_16_16_t min;
+
+ min.val = min(min1.val, min2.val);
+ return min;
+}
+
+static inline uint_fixed_16_16_t max_fixed_16_16(uint_fixed_16_16_t max1,
+ uint_fixed_16_16_t max2)
+{
+ uint_fixed_16_16_t max;
+
+ max.val = max(max1.val, max2.val);
+ return max;
+}
+
+static inline uint_fixed_16_16_t fixed_16_16_div_round_up(uint32_t val,
+ uint32_t d)
+{
+ uint_fixed_16_16_t fp, res;
+
+ fp = u32_to_fixed_16_16(val);
+ res.val = DIV_ROUND_UP(fp.val, d);
+ return res;
+}
+
+static inline uint_fixed_16_16_t fixed_16_16_div_round_up_u64(uint32_t val,
+ uint32_t d)
+{
+ uint_fixed_16_16_t res;
+ uint64_t interm_val;
+
+ interm_val = (uint64_t)val << 16;
+ interm_val = DIV_ROUND_UP_ULL(interm_val, d);
+ WARN_ON(interm_val >> 32);
+ res.val = (uint32_t) interm_val;
+
+ return res;
+}
+
+static inline uint_fixed_16_16_t mul_u32_fixed_16_16(uint32_t val,
+ uint_fixed_16_16_t mul)
+{
+ uint64_t intermediate_val;
+ uint_fixed_16_16_t fp;
+
+ intermediate_val = (uint64_t) val * mul.val;
+ WARN_ON(intermediate_val >> 32);
+ fp.val = (uint32_t) intermediate_val;
+ return fp;
+}
+
static inline const char *yesno(bool v)
{
return v ? "yes" : "no";
}
/*
+ * Global legacy plane identifier. Valid only for primary/sprite
+ * planes on pre-g4x, and only for primary planes on g4x+.
+ */
+enum plane {
+ PLANE_A,
+ PLANE_B,
+ PLANE_C,
+};
+#define plane_name(p) ((p) + 'A')
+
+#define sprite_name(p, s) ((p) * INTEL_INFO(dev_priv)->num_sprites[(p)] + (s) + 'A')
+
+/*
+ * Per-pipe plane identifier.
* I915_MAX_PLANES in the enum below is the maximum (across all platforms)
* number of planes per CRTC. Not all platforms really have this many planes,
* which means some arrays of size I915_MAX_PLANES may have unused entries
* between the topmost sprite plane and the cursor plane.
+ *
+ * This is expected to be passed to various register macros
+ * (eg. PLANE_CTL(), PS_PLANE_SEL(), etc.) so adjust with care.
*/
-enum plane {
- PLANE_A = 0,
- PLANE_B,
- PLANE_C,
+enum plane_id {
+ PLANE_PRIMARY,
+ PLANE_SPRITE0,
+ PLANE_SPRITE1,
+ PLANE_SPRITE2,
PLANE_CURSOR,
I915_MAX_PLANES,
};
-#define plane_name(p) ((p) + 'A')
-#define sprite_name(p, s) ((p) * INTEL_INFO(dev_priv)->num_sprites[(p)] + (s) + 'A')
+#define for_each_plane_id_on_crtc(__crtc, __p) \
+ for ((__p) = PLANE_PRIMARY; (__p) < I915_MAX_PLANES; (__p)++) \
+ for_each_if ((__crtc)->plane_ids_mask & BIT(__p))
enum port {
PORT_NONE = -1,
enum dpio_phy {
DPIO_PHY0,
- DPIO_PHY1
+ DPIO_PHY1,
+ DPIO_PHY2,
};
enum intel_display_power_domain {
POWER_DOMAIN_PORT_DDI_C_LANES,
POWER_DOMAIN_PORT_DDI_D_LANES,
POWER_DOMAIN_PORT_DDI_E_LANES,
+ POWER_DOMAIN_PORT_DDI_A_IO,
+ POWER_DOMAIN_PORT_DDI_B_IO,
+ POWER_DOMAIN_PORT_DDI_C_IO,
+ POWER_DOMAIN_PORT_DDI_D_IO,
+ POWER_DOMAIN_PORT_DDI_E_IO,
POWER_DOMAIN_PORT_DSI,
POWER_DOMAIN_PORT_CRT,
POWER_DOMAIN_PORT_OTHER,
#define for_each_hpd_pin(__pin) \
for ((__pin) = (HPD_NONE + 1); (__pin) < HPD_NUM_PINS; (__pin)++)
+#define HPD_STORM_DEFAULT_THRESHOLD 5
+
struct i915_hotplug {
struct work_struct hotplug_work;
struct work_struct poll_init_work;
bool poll_enabled;
+ unsigned int hpd_storm_threshold;
+
/*
* if we get a HPD irq from DP and a HPD irq from non-DP
* the non-DP HPD could block the workqueue on a mode config
&(dev)->mode_config.encoder_list, \
base.head)
-#define for_each_intel_connector(dev, intel_connector) \
- list_for_each_entry(intel_connector, \
- &(dev)->mode_config.connector_list, \
- base.head)
+#define for_each_intel_connector_iter(intel_connector, iter) \
+ while ((intel_connector = to_intel_connector(drm_connector_list_iter_next(iter))))
#define for_each_encoder_on_crtc(dev, __crtc, intel_encoder) \
list_for_each_entry((intel_encoder), &(dev)->mode_config.encoder_list, base.head) \
#define for_each_power_domain(domain, mask) \
for ((domain) = 0; (domain) < POWER_DOMAIN_NUM; (domain)++) \
- for_each_if ((1 << (domain)) & (mask))
+ for_each_if (BIT_ULL(domain) & (mask))
+
+#define for_each_power_well(__dev_priv, __power_well) \
+ for ((__power_well) = (__dev_priv)->power_domains.power_wells; \
+ (__power_well) - (__dev_priv)->power_domains.power_wells < \
+ (__dev_priv)->power_domains.power_well_count; \
+ (__power_well)++)
+
+#define for_each_power_well_rev(__dev_priv, __power_well) \
+ for ((__power_well) = (__dev_priv)->power_domains.power_wells + \
+ (__dev_priv)->power_domains.power_well_count - 1; \
+ (__power_well) - (__dev_priv)->power_domains.power_wells >= 0; \
+ (__power_well)--)
+
+#define for_each_power_domain_well(__dev_priv, __power_well, __domain_mask) \
+ for_each_power_well(__dev_priv, __power_well) \
+ for_each_if ((__power_well)->domains & (__domain_mask))
+
+#define for_each_power_domain_well_rev(__dev_priv, __power_well, __domain_mask) \
+ for_each_power_well_rev(__dev_priv, __power_well) \
+ for_each_if ((__power_well)->domains & (__domain_mask))
+
+#define for_each_intel_plane_in_state(__state, plane, plane_state, __i) \
+ for ((__i) = 0; \
+ (__i) < (__state)->base.dev->mode_config.num_total_plane && \
+ ((plane) = to_intel_plane((__state)->base.planes[__i].ptr), \
+ (plane_state) = to_intel_plane_state((__state)->base.planes[__i].state), 1); \
+ (__i)++) \
+ for_each_if (plane_state)
struct drm_i915_private;
struct i915_mm_struct;
} rps;
unsigned int bsd_engine;
+
+/* Client can have a maximum of 3 contexts banned before
+ * it is denied of creating new contexts. As one context
+ * ban needs 4 consecutive hangs, and more if there is
+ * progress in between, this is a last resort stop gap measure
+ * to limit the badly behaving clients access to gpu.
+ */
+#define I915_MAX_CLIENT_CONTEXT_BANS 3
+ int context_bans;
};
/* Used by dp and fdi links */
struct intel_crtc;
struct intel_limit;
struct dpll;
+struct intel_cdclk_state;
struct drm_i915_display_funcs {
- int (*get_display_clock_speed)(struct drm_i915_private *dev_priv);
+ void (*get_cdclk)(struct drm_i915_private *dev_priv,
+ struct intel_cdclk_state *cdclk_state);
+ void (*set_cdclk)(struct drm_i915_private *dev_priv,
+ const struct intel_cdclk_state *cdclk_state);
int (*get_fifo_size)(struct drm_i915_private *dev_priv, int plane);
int (*compute_pipe_wm)(struct intel_crtc_state *cstate);
int (*compute_intermediate_wm)(struct drm_device *dev,
int (*compute_global_watermarks)(struct drm_atomic_state *state);
void (*update_wm)(struct intel_crtc *crtc);
int (*modeset_calc_cdclk)(struct drm_atomic_state *state);
- void (*modeset_commit_cdclk)(struct drm_atomic_state *state);
/* Returns the active state of the crtc, and if the crtc is active,
* fills out the pipe-config with the hw state. */
bool (*get_pipe_config)(struct intel_crtc *,
struct intel_encoder *encoder,
const struct drm_display_mode *adjusted_mode);
void (*audio_codec_disable)(struct intel_encoder *encoder);
- void (*fdi_link_train)(struct drm_crtc *crtc);
+ void (*fdi_link_train)(struct intel_crtc *crtc,
+ const struct intel_crtc_state *crtc_state);
void (*init_clock_gating)(struct drm_i915_private *dev_priv);
int (*queue_flip)(struct drm_device *dev, struct drm_crtc *crtc,
struct drm_framebuffer *fb,
const struct intel_forcewake_range *fw_domains_table;
unsigned int fw_domains_table_entries;
+ struct notifier_block pmic_bus_access_nb;
struct intel_uncore_funcs funcs;
unsigned fifo_count;
};
#define DEV_INFO_FOR_EACH_FLAG(func) \
- /* Keep is_* in chronological order */ \
func(is_mobile); \
- func(is_i85x); \
- func(is_i915g); \
- func(is_i945gm); \
- func(is_g33); \
- func(is_g4x); \
- func(is_pineview); \
- func(is_broadwater); \
- func(is_crestline); \
- func(is_ivybridge); \
- func(is_valleyview); \
- func(is_cherryview); \
- func(is_haswell); \
- func(is_broadwell); \
- func(is_skylake); \
- func(is_broxton); \
- func(is_kabylake); \
+ func(is_lp); \
func(is_alpha_support); \
/* Keep has_* in alphabetical order */ \
func(has_64bit_reloc); \
+ func(has_aliasing_ppgtt); \
func(has_csr); \
func(has_ddi); \
+ func(has_decoupled_mmio); \
func(has_dp_mst); \
func(has_fbc); \
func(has_fpga_dbg); \
+ func(has_full_ppgtt); \
+ func(has_full_48bit_ppgtt); \
func(has_gmbus_irq); \
func(has_gmch_display); \
func(has_guc); \
func(cursor_needs_physical); \
func(hws_needs_physical); \
func(overlay_needs_physical); \
- func(supports_tv); \
- func(has_decoupled_mmio)
+ func(supports_tv);
struct sseu_dev_info {
u8 slice_mask;
return hweight8(sseu->slice_mask) * hweight8(sseu->subslice_mask);
}
+/* Keep in gen based order, and chronological order within a gen */
+enum intel_platform {
+ INTEL_PLATFORM_UNINITIALIZED = 0,
+ INTEL_I830,
+ INTEL_I845G,
+ INTEL_I85X,
+ INTEL_I865G,
+ INTEL_I915G,
+ INTEL_I915GM,
+ INTEL_I945G,
+ INTEL_I945GM,
+ INTEL_G33,
+ INTEL_PINEVIEW,
+ INTEL_I965G,
+ INTEL_I965GM,
+ INTEL_G45,
+ INTEL_GM45,
+ INTEL_IRONLAKE,
+ INTEL_SANDYBRIDGE,
+ INTEL_IVYBRIDGE,
+ INTEL_VALLEYVIEW,
+ INTEL_HASWELL,
+ INTEL_BROADWELL,
+ INTEL_CHERRYVIEW,
+ INTEL_SKYLAKE,
+ INTEL_BROXTON,
+ INTEL_KABYLAKE,
+ INTEL_GEMINILAKE,
+ INTEL_MAX_PLATFORMS
+};
+
struct intel_device_info {
u32 display_mmio_offset;
u16 device_id;
u8 num_pipes;
u8 num_sprites[I915_MAX_PIPES];
+ u8 num_scalers[I915_MAX_PIPES];
u8 gen;
u16 gen_mask;
+ enum intel_platform platform;
u8 ring_mask; /* Rings supported by the HW */
u8 num_rings;
#define DEFINE_FLAG(name) u8 name:1
struct intel_display_error_state;
-struct drm_i915_error_state {
+struct i915_gpu_state {
struct kref ref;
struct timeval time;
struct timeval boottime;
char error_msg[128];
bool simulated;
+ bool awake;
+ bool wakelock;
+ bool suspended;
int iommu;
u32 reset_count;
u32 suspend_count;
struct intel_device_info device_info;
+ struct i915_params params;
/* Generic register state */
u32 eir;
u32 pgtbl_er;
u32 ier;
- u32 gtier[4];
+ u32 gtier[4], ngtier;
u32 ccid;
u32 derrmr;
u32 forcewake;
u32 gab_ctl;
u32 gfx_mode;
+ u32 nfence;
u64 fence[I915_MAX_NUM_FENCES];
struct intel_overlay_error_state *overlay;
struct intel_display_error_state *display;
/* Software tracked state */
bool waiting;
int num_waiters;
- int hangcheck_score;
+ unsigned long hangcheck_timestamp;
+ bool hangcheck_stalled;
enum intel_engine_hangcheck_action hangcheck_action;
struct i915_address_space *vm;
int num_requests;
u32 semaphore_mboxes[I915_NUM_ENGINES - 1];
struct intel_instdone instdone;
+ struct drm_i915_error_context {
+ char comm[TASK_COMM_LEN];
+ pid_t pid;
+ u32 handle;
+ u32 hw_id;
+ int ban_score;
+ int active;
+ int guilty;
+ } context;
+
struct drm_i915_error_object {
u64 gtt_offset;
u64 gtt_size;
long jiffies;
pid_t pid;
u32 context;
+ int ban_score;
u32 seqno;
u32 head;
u32 tail;
u32 pp_dir_base;
};
} vm_info;
-
- pid_t pid;
- char comm[TASK_COMM_LEN];
} engine[I915_NUM_ENGINES];
struct drm_i915_error_buffer {
I915_CACHE_WT, /* hsw:gt3e WriteThrough for scanouts */
};
-struct i915_ctx_hang_stats {
- /* This context had batch pending when hang was declared */
- unsigned batch_pending;
-
- /* This context had batch active when hang was declared */
- unsigned batch_active;
-
- /* Time when this context was last blamed for a GPU reset */
- unsigned long guilty_ts;
-
- /* If the contexts causes a second GPU hang within this time,
- * it is permanently banned from submitting any more work.
- */
- unsigned long ban_period_seconds;
-
- /* This context is banned to submit more work */
- bool banned;
-};
-
-/* This must match up with the value previously used for execbuf2.rsvd1. */
-#define DEFAULT_CONTEXT_HANDLE 0
-
-/**
- * struct i915_gem_context - as the name implies, represents a context.
- * @ref: reference count.
- * @user_handle: userspace tracking identity for this context.
- * @remap_slice: l3 row remapping information.
- * @flags: context specific flags:
- * CONTEXT_NO_ZEROMAP: do not allow mapping things to page 0.
- * @file_priv: filp associated with this context (NULL for global default
- * context).
- * @hang_stats: information about the role of this context in possible GPU
- * hangs.
- * @ppgtt: virtual memory space used by this context.
- * @legacy_hw_ctx: render context backing object and whether it is correctly
- * initialized (legacy ring submission mechanism only).
- * @link: link in the global list of contexts.
- *
- * Contexts are memory images used by the hardware to store copies of their
- * internal state.
- */
-struct i915_gem_context {
- struct kref ref;
- struct drm_i915_private *i915;
- struct drm_i915_file_private *file_priv;
- struct i915_hw_ppgtt *ppgtt;
- struct pid *pid;
- const char *name;
-
- struct i915_ctx_hang_stats hang_stats;
-
- unsigned long flags;
-#define CONTEXT_NO_ZEROMAP BIT(0)
-#define CONTEXT_NO_ERROR_CAPTURE BIT(1)
-
- /* Unique identifier for this context, used by the hw for tracking */
- unsigned int hw_id;
- u32 user_handle;
- int priority; /* greater priorities are serviced first */
-
- u32 ggtt_alignment;
-
- struct intel_context {
- struct i915_vma *state;
- struct intel_ring *ring;
- uint32_t *lrc_reg_state;
- u64 lrc_desc;
- int pin_count;
- bool initialised;
- } engine[I915_NUM_ENGINES];
- u32 ring_size;
- u32 desc_template;
- struct atomic_notifier_head status_notifier;
- bool execlists_force_single_submission;
-
- struct list_head link;
-
- u8 remap_slice;
- bool closed:1;
-};
+#define I915_COLOR_UNEVICTABLE (-1) /* a non-vma sharing the address space */
enum fb_op_origin {
ORIGIN_GTT,
} plane;
struct {
- uint32_t pixel_format;
+ const struct drm_format_info *format;
unsigned int stride;
} fb;
} state_cache;
} crtc;
struct {
- uint32_t pixel_format;
+ const struct drm_format_info *format;
unsigned int stride;
} fb;
const char *no_fbc_reason;
};
-/**
+/*
* HIGH_RR is the highest eDP panel refresh rate read from EDID
* LOW_RR is the lowest eDP panel refresh rate found from EDID
* parsing for same resolution.
bool psr2_support;
bool aux_frame_sync;
bool link_standby;
+ bool y_cord_support;
+ bool colorimetry_support;
+ bool alpm;
};
enum intel_pch {
u32 pm_iir;
/* PM interrupt bits that should never be masked */
- u32 pm_intr_keep;
+ u32 pm_intrmsk_mbz;
/* Frequencies are stored in potentially platform dependent multiples.
* In other words, *_freq needs to be multiplied by X to be interesting.
unsigned boosts;
/* manual wa residency calculations */
- struct intel_rps_ei up_ei, down_ei;
+ struct intel_rps_ei ei;
/*
* Protects RPS/RC6 register access and PCU communication.
int count;
/* cached hw enabled state */
bool hw_enabled;
- unsigned long domains;
+ u64 domains;
/* unique identifier for this power well */
unsigned long id;
/*
struct work_struct free_work;
/** Usable portion of the GTT for GEM */
- unsigned long stolen_base; /* limited to low memory (32-bit) */
+ dma_addr_t stolen_base; /* limited to low memory (32-bit) */
/** PPGTT used for aliasing the PPGTT with the GTT */
struct i915_hw_ppgtt *aliasing_ppgtt;
loff_t pos;
};
-struct i915_error_state_file_priv {
- struct drm_device *dev;
- struct drm_i915_error_state *error;
-};
-
#define I915_RESET_TIMEOUT (10 * HZ) /* 10s */
#define I915_FENCE_TIMEOUT (10 * HZ) /* 10s */
+#define I915_ENGINE_DEAD_TIMEOUT (4 * HZ) /* Seqno, head and subunits dead */
+#define I915_SEQNO_DEAD_TIMEOUT (12 * HZ) /* Seqno dead with active head */
+
struct i915_gpu_error {
/* For hangcheck timer */
#define DRM_I915_HANGCHECK_PERIOD 1500 /* in ms */
#define DRM_I915_HANGCHECK_JIFFIES msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD)
- /* Hang gpu twice in this window and your context gets banned */
-#define DRM_I915_CTX_BAN_PERIOD DIV_ROUND_UP(8*DRM_I915_HANGCHECK_PERIOD, 1000)
struct delayed_work hangcheck_work;
/* For reset and error_state handling. */
spinlock_t lock;
/* Protected by the above dev->gpu_error.lock. */
- struct drm_i915_error_state *first_error;
+ struct i915_gpu_state *first_error;
unsigned long missed_irq_rings;
uint8_t supports_dvi:1;
uint8_t supports_hdmi:1;
uint8_t supports_dp:1;
+ uint8_t supports_edp:1;
uint8_t alternate_aux_channel;
uint8_t alternate_ddc_pin;
bool present;
bool active_low_pwm;
u8 min_brightness; /* min_brightness/255 of max */
+ u8 controller; /* brightness controller number */
enum intel_backlight_type type;
} backlight;
};
struct vlv_pipe_wm {
- uint16_t primary;
- uint16_t sprite[2];
- uint8_t cursor;
+ uint16_t plane[I915_MAX_PLANES];
};
struct vlv_sr_wm {
uint16_t plane;
- uint8_t cursor;
+ uint16_t cursor;
+};
+
+struct vlv_wm_ddl_values {
+ uint8_t plane[I915_MAX_PLANES];
};
struct vlv_wm_values {
struct vlv_pipe_wm pipe[3];
struct vlv_sr_wm sr;
- struct {
- uint8_t cursor;
- uint8_t sprite[2];
- uint8_t primary;
- } ddl[3];
+ struct vlv_wm_ddl_values ddl[3];
uint8_t level;
bool cxsr;
};
enum intel_pipe_crc_source source;
int head, tail;
wait_queue_head_t wq;
+ int skipped;
};
struct i915_frontbuffer_tracking {
bool sprites_scaled;
};
+struct i915_oa_format {
+ u32 format;
+ int size;
+};
+
+struct i915_oa_reg {
+ i915_reg_t addr;
+ u32 value;
+};
+
+struct i915_perf_stream;
+
+/**
+ * struct i915_perf_stream_ops - the OPs to support a specific stream type
+ */
+struct i915_perf_stream_ops {
+ /**
+ * @enable: Enables the collection of HW samples, either in response to
+ * `I915_PERF_IOCTL_ENABLE` or implicitly called when stream is opened
+ * without `I915_PERF_FLAG_DISABLED`.
+ */
+ void (*enable)(struct i915_perf_stream *stream);
+
+ /**
+ * @disable: Disables the collection of HW samples, either in response
+ * to `I915_PERF_IOCTL_DISABLE` or implicitly called before destroying
+ * the stream.
+ */
+ void (*disable)(struct i915_perf_stream *stream);
+
+ /**
+ * @poll_wait: Call poll_wait, passing a wait queue that will be woken
+ * once there is something ready to read() for the stream
+ */
+ void (*poll_wait)(struct i915_perf_stream *stream,
+ struct file *file,
+ poll_table *wait);
+
+ /**
+ * @wait_unlocked: For handling a blocking read, wait until there is
+ * something to ready to read() for the stream. E.g. wait on the same
+ * wait queue that would be passed to poll_wait().
+ */
+ int (*wait_unlocked)(struct i915_perf_stream *stream);
+
+ /**
+ * @read: Copy buffered metrics as records to userspace
+ * **buf**: the userspace, destination buffer
+ * **count**: the number of bytes to copy, requested by userspace
+ * **offset**: zero at the start of the read, updated as the read
+ * proceeds, it represents how many bytes have been copied so far and
+ * the buffer offset for copying the next record.
+ *
+ * Copy as many buffered i915 perf samples and records for this stream
+ * to userspace as will fit in the given buffer.
+ *
+ * Only write complete records; returning -%ENOSPC if there isn't room
+ * for a complete record.
+ *
+ * Return any error condition that results in a short read such as
+ * -%ENOSPC or -%EFAULT, even though these may be squashed before
+ * returning to userspace.
+ */
+ int (*read)(struct i915_perf_stream *stream,
+ char __user *buf,
+ size_t count,
+ size_t *offset);
+
+ /**
+ * @destroy: Cleanup any stream specific resources.
+ *
+ * The stream will always be disabled before this is called.
+ */
+ void (*destroy)(struct i915_perf_stream *stream);
+};
+
+/**
+ * struct i915_perf_stream - state for a single open stream FD
+ */
+struct i915_perf_stream {
+ /**
+ * @dev_priv: i915 drm device
+ */
+ struct drm_i915_private *dev_priv;
+
+ /**
+ * @link: Links the stream into ``&drm_i915_private->streams``
+ */
+ struct list_head link;
+
+ /**
+ * @sample_flags: Flags representing the `DRM_I915_PERF_PROP_SAMPLE_*`
+ * properties given when opening a stream, representing the contents
+ * of a single sample as read() by userspace.
+ */
+ u32 sample_flags;
+
+ /**
+ * @sample_size: Considering the configured contents of a sample
+ * combined with the required header size, this is the total size
+ * of a single sample record.
+ */
+ int sample_size;
+
+ /**
+ * @ctx: %NULL if measuring system-wide across all contexts or a
+ * specific context that is being monitored.
+ */
+ struct i915_gem_context *ctx;
+
+ /**
+ * @enabled: Whether the stream is currently enabled, considering
+ * whether the stream was opened in a disabled state and based
+ * on `I915_PERF_IOCTL_ENABLE` and `I915_PERF_IOCTL_DISABLE` calls.
+ */
+ bool enabled;
+
+ /**
+ * @ops: The callbacks providing the implementation of this specific
+ * type of configured stream.
+ */
+ const struct i915_perf_stream_ops *ops;
+};
+
+/**
+ * struct i915_oa_ops - Gen specific implementation of an OA unit stream
+ */
+struct i915_oa_ops {
+ /**
+ * @init_oa_buffer: Resets the head and tail pointers of the
+ * circular buffer for periodic OA reports.
+ *
+ * Called when first opening a stream for OA metrics, but also may be
+ * called in response to an OA buffer overflow or other error
+ * condition.
+ *
+ * Note it may be necessary to clear the full OA buffer here as part of
+ * maintaining the invariable that new reports must be written to
+ * zeroed memory for us to be able to reliable detect if an expected
+ * report has not yet landed in memory. (At least on Haswell the OA
+ * buffer tail pointer is not synchronized with reports being visible
+ * to the CPU)
+ */
+ void (*init_oa_buffer)(struct drm_i915_private *dev_priv);
+
+ /**
+ * @enable_metric_set: Applies any MUX configuration to set up the
+ * Boolean and Custom (B/C) counters that are part of the counter
+ * reports being sampled. May apply system constraints such as
+ * disabling EU clock gating as required.
+ */
+ int (*enable_metric_set)(struct drm_i915_private *dev_priv);
+
+ /**
+ * @disable_metric_set: Remove system constraints associated with using
+ * the OA unit.
+ */
+ void (*disable_metric_set)(struct drm_i915_private *dev_priv);
+
+ /**
+ * @oa_enable: Enable periodic sampling
+ */
+ void (*oa_enable)(struct drm_i915_private *dev_priv);
+
+ /**
+ * @oa_disable: Disable periodic sampling
+ */
+ void (*oa_disable)(struct drm_i915_private *dev_priv);
+
+ /**
+ * @read: Copy data from the circular OA buffer into a given userspace
+ * buffer.
+ */
+ int (*read)(struct i915_perf_stream *stream,
+ char __user *buf,
+ size_t count,
+ size_t *offset);
+
+ /**
+ * @oa_buffer_is_empty: Check if OA buffer empty (false positives OK)
+ *
+ * This is either called via fops or the poll check hrtimer (atomic
+ * ctx) without any locks taken.
+ *
+ * It's safe to read OA config state here unlocked, assuming that this
+ * is only called while the stream is enabled, while the global OA
+ * configuration can't be modified.
+ *
+ * Efficiency is more important than avoiding some false positives
+ * here, which will be handled gracefully - likely resulting in an
+ * %EAGAIN error for userspace.
+ */
+ bool (*oa_buffer_is_empty)(struct drm_i915_private *dev_priv);
+};
+
+struct intel_cdclk_state {
+ unsigned int cdclk, vco, ref;
+};
+
struct drm_i915_private {
struct drm_device drm;
const struct intel_device_info info;
- int relative_constants_mode;
-
void __iomem *regs;
struct intel_uncore uncore;
struct intel_gvt *gvt;
+ struct intel_huc huc;
struct intel_guc guc;
struct intel_csr csr;
unsigned int fsb_freq, mem_freq, is_ddr3;
unsigned int skl_preferred_vco_freq;
- unsigned int cdclk_freq, max_cdclk_freq, atomic_cdclk_freq;
+ unsigned int max_cdclk_freq;
+
unsigned int max_dotclk_freq;
unsigned int rawclk_freq;
unsigned int hpll_freq;
unsigned int czclk_freq;
struct {
- unsigned int vco, ref;
- } cdclk_pll;
+ /*
+ * The current logical cdclk state.
+ * See intel_atomic_state.cdclk.logical
+ *
+ * For reading holding any crtc lock is sufficient,
+ * for writing must hold all of them.
+ */
+ struct intel_cdclk_state logical;
+ /*
+ * The current actual cdclk state.
+ * See intel_atomic_state.cdclk.actual
+ */
+ struct intel_cdclk_state actual;
+ /* The current hardware cdclk state */
+ struct intel_cdclk_state hw;
+ } cdclk;
/**
* wq - Driver workqueue for GEM.
} sagv_status;
struct {
+ /* protects DSPARB registers on pre-g4x/vlv/chv */
+ spinlock_t dsparb_lock;
+
/*
* Raw watermark latency values:
* in 0.1us units for WM0,
struct i915_runtime_pm pm;
+ struct {
+ bool initialized;
+
+ struct kobject *metrics_kobj;
+ struct ctl_table_header *sysctl_header;
+
+ struct mutex lock;
+ struct list_head streams;
+
+ spinlock_t hook_lock;
+
+ struct {
+ struct i915_perf_stream *exclusive_stream;
+
+ u32 specific_ctx_id;
+
+ struct hrtimer poll_check_timer;
+ wait_queue_head_t poll_wq;
+ bool pollin;
+
+ bool periodic;
+ int period_exponent;
+ int timestamp_frequency;
+
+ int tail_margin;
+
+ int metrics_set;
+
+ const struct i915_oa_reg *mux_regs;
+ int mux_regs_len;
+ const struct i915_oa_reg *b_counter_regs;
+ int b_counter_regs_len;
+
+ struct {
+ struct i915_vma *vma;
+ u8 *vaddr;
+ int format;
+ int format_size;
+ } oa_buffer;
+
+ u32 gen7_latched_oastatus1;
+
+ struct i915_oa_ops ops;
+ const struct i915_oa_format *oa_formats;
+ int n_builtin_sets;
+ } oa;
+ } perf;
+
/* Abstract the submission mechanism (legacy ringbuffer or execlists) away */
struct {
void (*resume)(struct drm_i915_private *);
/* perform PHY state sanity checks? */
bool chv_phy_assert[2];
+ bool ipc_enabled;
+
/* Used to save the pipe-to-encoder mapping for audio */
struct intel_encoder *av_enc_map[I915_MAX_PIPES];
+ /* necessary resource sharing with HDMI LPE audio driver. */
+ struct {
+ struct platform_device *platdev;
+ int irq;
+ } lpe_audio;
+
/*
* NOTE: This is the dri1/ums dungeon, don't add stuff here. Your patch
* will be rejected. Instead look for a better place.
(((__iter).curr += PAGE_SIZE) < (__iter).max) || \
((__iter) = __sgt_iter(__sg_next((__iter).sgp), false), 0))
-/*
- * A command that requires special handling by the command parser.
- */
-struct drm_i915_cmd_descriptor {
- /*
- * Flags describing how the command parser processes the command.
- *
- * CMD_DESC_FIXED: The command has a fixed length if this is set,
- * a length mask if not set
- * CMD_DESC_SKIP: The command is allowed but does not follow the
- * standard length encoding for the opcode range in
- * which it falls
- * CMD_DESC_REJECT: The command is never allowed
- * CMD_DESC_REGISTER: The command should be checked against the
- * register whitelist for the appropriate ring
- * CMD_DESC_MASTER: The command is allowed if the submitting process
- * is the DRM master
- */
- u32 flags;
-#define CMD_DESC_FIXED (1<<0)
-#define CMD_DESC_SKIP (1<<1)
-#define CMD_DESC_REJECT (1<<2)
-#define CMD_DESC_REGISTER (1<<3)
-#define CMD_DESC_BITMASK (1<<4)
-#define CMD_DESC_MASTER (1<<5)
-
- /*
- * The command's unique identification bits and the bitmask to get them.
- * This isn't strictly the opcode field as defined in the spec and may
- * also include type, subtype, and/or subop fields.
- */
- struct {
- u32 value;
- u32 mask;
- } cmd;
-
- /*
- * The command's length. The command is either fixed length (i.e. does
- * not include a length field) or has a length field mask. The flag
- * CMD_DESC_FIXED indicates a fixed length. Otherwise, the command has
- * a length mask. All command entries in a command table must include
- * length information.
- */
- union {
- u32 fixed;
- u32 mask;
- } length;
-
- /*
- * Describes where to find a register address in the command to check
- * against the ring's register whitelist. Only valid if flags has the
- * CMD_DESC_REGISTER bit set.
- *
- * A non-zero step value implies that the command may access multiple
- * registers in sequence (e.g. LRI), in that case step gives the
- * distance in dwords between individual offset fields.
- */
- struct {
- u32 offset;
- u32 mask;
- u32 step;
- } reg;
-
-#define MAX_CMD_DESC_BITMASKS 3
- /*
- * Describes command checks where a particular dword is masked and
- * compared against an expected value. If the command does not match
- * the expected value, the parser rejects it. Only valid if flags has
- * the CMD_DESC_BITMASK bit set. Only entries where mask is non-zero
- * are valid.
- *
- * If the check specifies a non-zero condition_mask then the parser
- * only performs the check when the bits specified by condition_mask
- * are non-zero.
- */
- struct {
- u32 offset;
- u32 mask;
- u32 expected;
- u32 condition_offset;
- u32 condition_mask;
- } bits[MAX_CMD_DESC_BITMASKS];
-};
-
-/*
- * A table of commands requiring special handling by the command parser.
- *
- * Each engine has an array of tables. Each table consists of an array of
- * command descriptors, which must be sorted with command opcodes in
- * ascending order.
- */
-struct drm_i915_cmd_table {
- const struct drm_i915_cmd_descriptor *table;
- int count;
-};
-
static inline const struct intel_device_info *
intel_info(const struct drm_i915_private *dev_priv)
{
#define IS_REVID(p, since, until) \
(INTEL_REVID(p) >= (since) && INTEL_REVID(p) <= (until))
-#define IS_I830(dev_priv) (INTEL_DEVID(dev_priv) == 0x3577)
-#define IS_845G(dev_priv) (INTEL_DEVID(dev_priv) == 0x2562)
-#define IS_I85X(dev_priv) ((dev_priv)->info.is_i85x)
-#define IS_I865G(dev_priv) (INTEL_DEVID(dev_priv) == 0x2572)
-#define IS_I915G(dev_priv) ((dev_priv)->info.is_i915g)
-#define IS_I915GM(dev_priv) (INTEL_DEVID(dev_priv) == 0x2592)
-#define IS_I945G(dev_priv) (INTEL_DEVID(dev_priv) == 0x2772)
-#define IS_I945GM(dev_priv) ((dev_priv)->info.is_i945gm)
-#define IS_BROADWATER(dev_priv) ((dev_priv)->info.is_broadwater)
-#define IS_CRESTLINE(dev_priv) ((dev_priv)->info.is_crestline)
-#define IS_GM45(dev_priv) (INTEL_DEVID(dev_priv) == 0x2A42)
-#define IS_G4X(dev_priv) ((dev_priv)->info.is_g4x)
+#define IS_I830(dev_priv) ((dev_priv)->info.platform == INTEL_I830)
+#define IS_I845G(dev_priv) ((dev_priv)->info.platform == INTEL_I845G)
+#define IS_I85X(dev_priv) ((dev_priv)->info.platform == INTEL_I85X)
+#define IS_I865G(dev_priv) ((dev_priv)->info.platform == INTEL_I865G)
+#define IS_I915G(dev_priv) ((dev_priv)->info.platform == INTEL_I915G)
+#define IS_I915GM(dev_priv) ((dev_priv)->info.platform == INTEL_I915GM)
+#define IS_I945G(dev_priv) ((dev_priv)->info.platform == INTEL_I945G)
+#define IS_I945GM(dev_priv) ((dev_priv)->info.platform == INTEL_I945GM)
+#define IS_I965G(dev_priv) ((dev_priv)->info.platform == INTEL_I965G)
+#define IS_I965GM(dev_priv) ((dev_priv)->info.platform == INTEL_I965GM)
+#define IS_G45(dev_priv) ((dev_priv)->info.platform == INTEL_G45)
+#define IS_GM45(dev_priv) ((dev_priv)->info.platform == INTEL_GM45)
+#define IS_G4X(dev_priv) (IS_G45(dev_priv) || IS_GM45(dev_priv))
#define IS_PINEVIEW_G(dev_priv) (INTEL_DEVID(dev_priv) == 0xa001)
#define IS_PINEVIEW_M(dev_priv) (INTEL_DEVID(dev_priv) == 0xa011)
-#define IS_PINEVIEW(dev_priv) ((dev_priv)->info.is_pineview)
-#define IS_G33(dev_priv) ((dev_priv)->info.is_g33)
+#define IS_PINEVIEW(dev_priv) ((dev_priv)->info.platform == INTEL_PINEVIEW)
+#define IS_G33(dev_priv) ((dev_priv)->info.platform == INTEL_G33)
#define IS_IRONLAKE_M(dev_priv) (INTEL_DEVID(dev_priv) == 0x0046)
-#define IS_IVYBRIDGE(dev_priv) ((dev_priv)->info.is_ivybridge)
+#define IS_IVYBRIDGE(dev_priv) ((dev_priv)->info.platform == INTEL_IVYBRIDGE)
#define IS_IVB_GT1(dev_priv) (INTEL_DEVID(dev_priv) == 0x0156 || \
INTEL_DEVID(dev_priv) == 0x0152 || \
INTEL_DEVID(dev_priv) == 0x015a)
-#define IS_VALLEYVIEW(dev_priv) ((dev_priv)->info.is_valleyview)
-#define IS_CHERRYVIEW(dev_priv) ((dev_priv)->info.is_cherryview)
-#define IS_HASWELL(dev_priv) ((dev_priv)->info.is_haswell)
-#define IS_BROADWELL(dev_priv) ((dev_priv)->info.is_broadwell)
-#define IS_SKYLAKE(dev_priv) ((dev_priv)->info.is_skylake)
-#define IS_BROXTON(dev_priv) ((dev_priv)->info.is_broxton)
-#define IS_KABYLAKE(dev_priv) ((dev_priv)->info.is_kabylake)
+#define IS_VALLEYVIEW(dev_priv) ((dev_priv)->info.platform == INTEL_VALLEYVIEW)
+#define IS_CHERRYVIEW(dev_priv) ((dev_priv)->info.platform == INTEL_CHERRYVIEW)
+#define IS_HASWELL(dev_priv) ((dev_priv)->info.platform == INTEL_HASWELL)
+#define IS_BROADWELL(dev_priv) ((dev_priv)->info.platform == INTEL_BROADWELL)
+#define IS_SKYLAKE(dev_priv) ((dev_priv)->info.platform == INTEL_SKYLAKE)
+#define IS_BROXTON(dev_priv) ((dev_priv)->info.platform == INTEL_BROXTON)
+#define IS_KABYLAKE(dev_priv) ((dev_priv)->info.platform == INTEL_KABYLAKE)
+#define IS_GEMINILAKE(dev_priv) ((dev_priv)->info.platform == INTEL_GEMINILAKE)
#define IS_MOBILE(dev_priv) ((dev_priv)->info.is_mobile)
#define IS_HSW_EARLY_SDV(dev_priv) (IS_HASWELL(dev_priv) && \
(INTEL_DEVID(dev_priv) & 0xFF00) == 0x0C00)
#define BXT_REVID_A0 0x0
#define BXT_REVID_A1 0x1
#define BXT_REVID_B0 0x3
+#define BXT_REVID_B_LAST 0x8
#define BXT_REVID_C0 0x9
#define IS_BXT_REVID(dev_priv, since, until) \
#define IS_KBL_REVID(dev_priv, since, until) \
(IS_KABYLAKE(dev_priv) && IS_REVID(dev_priv, since, until))
+#define GLK_REVID_A0 0x0
+#define GLK_REVID_A1 0x1
+
+#define IS_GLK_REVID(dev_priv, since, until) \
+ (IS_GEMINILAKE(dev_priv) && IS_REVID(dev_priv, since, until))
+
/*
* The genX designation typically refers to the render engine, so render
* capability related checks should use IS_GEN, while display and other checks
#define IS_GEN8(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(7)))
#define IS_GEN9(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(8)))
+#define IS_LP(dev_priv) (INTEL_INFO(dev_priv)->is_lp)
+#define IS_GEN9_LP(dev_priv) (IS_GEN9(dev_priv) && IS_LP(dev_priv))
+#define IS_GEN9_BC(dev_priv) (IS_GEN9(dev_priv) && !IS_LP(dev_priv))
+
#define ENGINE_MASK(id) BIT(id)
#define RENDER_RING ENGINE_MASK(RCS)
#define BSD_RING ENGINE_MASK(VCS)
((dev_priv)->info.overlay_needs_physical)
/* Early gen2 have a totally busted CS tlb and require pinned batches. */
-#define HAS_BROKEN_CS_TLB(dev_priv) (IS_I830(dev_priv) || IS_845G(dev_priv))
+#define HAS_BROKEN_CS_TLB(dev_priv) (IS_I830(dev_priv) || IS_I845G(dev_priv))
/* WaRsDisableCoarsePowerGating:skl,bxt */
#define NEEDS_WaRsDisableCoarsePowerGating(dev_priv) \
- (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1) || \
- IS_SKL_GT3(dev_priv) || \
- IS_SKL_GT4(dev_priv))
+ (IS_SKL_GT3(dev_priv) || IS_SKL_GT4(dev_priv))
/*
* dp aux and gmbus irq on gen4 seems to be able to generate legacy interrupts
#define HAS_GUC(dev_priv) ((dev_priv)->info.has_guc)
#define HAS_GUC_UCODE(dev_priv) (HAS_GUC(dev_priv))
#define HAS_GUC_SCHED(dev_priv) (HAS_GUC(dev_priv))
+#define HAS_HUC_UCODE(dev_priv) (HAS_GUC(dev_priv))
#define HAS_RESOURCE_STREAMER(dev_priv) ((dev_priv)->info.has_resource_streamer)
return false;
}
-extern int i915_suspend_switcheroo(struct drm_device *dev, pm_message_t state);
-extern int i915_resume_switcheroo(struct drm_device *dev);
-
int intel_sanitize_enable_ppgtt(struct drm_i915_private *dev_priv,
int enable_ppgtt);
extern void i915_update_gfx_val(struct drm_i915_private *dev_priv);
int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool on);
+int intel_engines_init_early(struct drm_i915_private *dev_priv);
+int intel_engines_init(struct drm_i915_private *dev_priv);
+
/* intel_hotplug.c */
void intel_hpd_irq_handler(struct drm_i915_private *dev_priv,
u32 pin_mask, u32 long_mask);
void intel_irq_uninstall(struct drm_i915_private *dev_priv);
extern void intel_uncore_sanitize(struct drm_i915_private *dev_priv);
-extern void intel_uncore_early_sanitize(struct drm_i915_private *dev_priv,
- bool restore_forcewake);
extern void intel_uncore_init(struct drm_i915_private *dev_priv);
extern bool intel_uncore_unclaimed_mmio(struct drm_i915_private *dev_priv);
extern bool intel_uncore_arm_unclaimed_mmio_detection(struct drm_i915_private *dev_priv);
extern void intel_uncore_fini(struct drm_i915_private *dev_priv);
-extern void intel_uncore_forcewake_reset(struct drm_i915_private *dev_priv,
- bool restore);
+extern void intel_uncore_suspend(struct drm_i915_private *dev_priv);
+extern void intel_uncore_resume_early(struct drm_i915_private *dev_priv);
const char *intel_uncore_forcewake_domain_to_str(const enum forcewake_domain_id id);
void intel_uncore_forcewake_get(struct drm_i915_private *dev_priv,
enum forcewake_domains domains);
struct drm_file *file_priv);
int i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
struct drm_file *file_priv);
-int i915_gem_set_tiling(struct drm_device *dev, void *data,
- struct drm_file *file_priv);
-int i915_gem_get_tiling(struct drm_device *dev, void *data,
- struct drm_file *file_priv);
+int i915_gem_set_tiling_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *file_priv);
+int i915_gem_get_tiling_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *file_priv);
void i915_gem_init_userptr(struct drm_i915_private *dev_priv);
int i915_gem_userptr_ioctl(struct drm_device *dev, void *data,
struct drm_file *file);
struct drm_file *file_priv);
int i915_gem_wait_ioctl(struct drm_device *dev, void *data,
struct drm_file *file_priv);
-int i915_gem_load_init(struct drm_device *dev);
-void i915_gem_load_cleanup(struct drm_device *dev);
+void i915_gem_sanitize(struct drm_i915_private *i915);
+int i915_gem_load_init(struct drm_i915_private *dev_priv);
+void i915_gem_load_cleanup(struct drm_i915_private *dev_priv);
void i915_gem_load_init_fences(struct drm_i915_private *dev_priv);
int i915_gem_freeze(struct drm_i915_private *dev_priv);
int i915_gem_freeze_late(struct drm_i915_private *dev_priv);
-void *i915_gem_object_alloc(struct drm_device *dev);
+void *i915_gem_object_alloc(struct drm_i915_private *dev_priv);
void i915_gem_object_free(struct drm_i915_gem_object *obj);
void i915_gem_object_init(struct drm_i915_gem_object *obj,
const struct drm_i915_gem_object_ops *ops);
-struct drm_i915_gem_object *i915_gem_object_create(struct drm_device *dev,
- u64 size);
-struct drm_i915_gem_object *i915_gem_object_create_from_data(
- struct drm_device *dev, const void *data, size_t size);
+struct drm_i915_gem_object *
+i915_gem_object_create(struct drm_i915_private *dev_priv, u64 size);
+struct drm_i915_gem_object *
+i915_gem_object_create_from_data(struct drm_i915_private *dev_priv,
+ const void *data, size_t size);
void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file);
void i915_gem_free_object(struct drm_gem_object *obj);
+static inline void i915_gem_drain_freed_objects(struct drm_i915_private *i915)
+{
+ /* A single pass should suffice to release all the freed objects (along
+ * most call paths) , but be a little more paranoid in that freeing
+ * the objects does take a little amount of time, during which the rcu
+ * callbacks could have added new objects into the freed list, and
+ * armed the work again.
+ */
+ do {
+ rcu_barrier();
+ } while (flush_work(&i915->mm.free_work));
+}
+
struct i915_vma * __must_check
i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
const struct i915_ggtt_view *view,
GEM_BUG_ON(!obj->mm.pages);
atomic_dec(&obj->mm.pages_pin_count);
- GEM_BUG_ON(atomic_read(&obj->mm.pages_pin_count) < obj->bind_count);
}
static inline void
/**
* i915_gem_object_pin_map - return a contiguous mapping of the entire object
- * @obj - the object to map into kernel address space
- * @type - the type of mapping, used to select pgprot_t
+ * @obj: the object to map into kernel address space
+ * @type: the type of mapping, used to select pgprot_t
*
* Calls i915_gem_object_pin_pages() to prevent reaping of the object's
* pages and then returns a contiguous mapping of the backing storage into
/**
* i915_gem_object_unpin_map - releases an earlier mapping
- * @obj - the object to unmap
+ * @obj: the object to unmap
*
* After pinning the object and mapping its pages, once you are finished
* with your access, call i915_gem_object_unpin_map() to release the pin
return READ_ONCE(error->reset_count);
}
+int i915_gem_reset_prepare(struct drm_i915_private *dev_priv);
void i915_gem_reset(struct drm_i915_private *dev_priv);
+void i915_gem_reset_finish(struct drm_i915_private *dev_priv);
void i915_gem_set_wedged(struct drm_i915_private *dev_priv);
-void i915_gem_clflush_object(struct drm_i915_gem_object *obj, bool force);
-int __must_check i915_gem_init(struct drm_device *dev);
-int __must_check i915_gem_init_hw(struct drm_device *dev);
+
+void i915_gem_init_mmio(struct drm_i915_private *i915);
+int __must_check i915_gem_init(struct drm_i915_private *dev_priv);
+int __must_check i915_gem_init_hw(struct drm_i915_private *dev_priv);
void i915_gem_init_swizzling(struct drm_i915_private *dev_priv);
-void i915_gem_cleanup_engines(struct drm_device *dev);
-int __must_check i915_gem_wait_for_idle(struct drm_i915_private *dev_priv,
- unsigned int flags);
-int __must_check i915_gem_suspend(struct drm_device *dev);
-void i915_gem_resume(struct drm_device *dev);
-int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf);
+void i915_gem_cleanup_engines(struct drm_i915_private *dev_priv);
+int i915_gem_wait_for_idle(struct drm_i915_private *dev_priv,
+ unsigned int flags);
+int __must_check i915_gem_suspend(struct drm_i915_private *dev_priv);
+void i915_gem_resume(struct drm_i915_private *dev_priv);
+int i915_gem_fault(struct vm_fault *vmf);
int i915_gem_object_wait(struct drm_i915_gem_object *obj,
unsigned int flags,
long timeout,
int i915_gem_open(struct drm_device *dev, struct drm_file *file);
void i915_gem_release(struct drm_device *dev, struct drm_file *file);
-u64 i915_gem_get_ggtt_size(struct drm_i915_private *dev_priv, u64 size,
- int tiling_mode);
-u64 i915_gem_get_ggtt_alignment(struct drm_i915_private *dev_priv, u64 size,
- int tiling_mode, bool fenced);
-
int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
enum i915_cache_level cache_level);
struct dma_buf *i915_gem_prime_export(struct drm_device *dev,
struct drm_gem_object *gem_obj, int flags);
-struct i915_vma *
-i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
- struct i915_address_space *vm,
- const struct i915_ggtt_view *view);
-
-struct i915_vma *
-i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj,
- struct i915_address_space *vm,
- const struct i915_ggtt_view *view);
-
static inline struct i915_hw_ppgtt *
i915_vm_to_ppgtt(struct i915_address_space *vm)
{
return container_of(vm, struct i915_hw_ppgtt, base);
}
-static inline struct i915_vma *
-i915_gem_object_to_ggtt(struct drm_i915_gem_object *obj,
- const struct i915_ggtt_view *view)
-{
- return i915_gem_obj_to_vma(obj, &to_i915(obj->base.dev)->ggtt.base, view);
-}
-
/* i915_gem_fence_reg.c */
int __must_check i915_vma_get_fence(struct i915_vma *vma);
int __must_check i915_vma_put_fence(struct i915_vma *vma);
+void i915_gem_revoke_fences(struct drm_i915_private *dev_priv);
void i915_gem_restore_fences(struct drm_i915_private *dev_priv);
void i915_gem_detect_bit_6_swizzle(struct drm_i915_private *dev_priv);
void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj,
struct sg_table *pages);
-/* i915_gem_context.c */
-int __must_check i915_gem_context_init(struct drm_device *dev);
-void i915_gem_context_lost(struct drm_i915_private *dev_priv);
-void i915_gem_context_fini(struct drm_device *dev);
-int i915_gem_context_open(struct drm_device *dev, struct drm_file *file);
-void i915_gem_context_close(struct drm_device *dev, struct drm_file *file);
-int i915_switch_context(struct drm_i915_gem_request *req);
-int i915_gem_switch_to_kernel_context(struct drm_i915_private *dev_priv);
-struct i915_vma *
-i915_gem_context_pin_legacy(struct i915_gem_context *ctx,
- unsigned int flags);
-void i915_gem_context_free(struct kref *ctx_ref);
-struct drm_i915_gem_object *
-i915_gem_alloc_context_obj(struct drm_device *dev, size_t size);
-struct i915_gem_context *
-i915_gem_context_create_gvt(struct drm_device *dev);
-
static inline struct i915_gem_context *
i915_gem_context_lookup(struct drm_i915_file_private *file_priv, u32 id)
{
kref_put(&ctx->ref, i915_gem_context_free);
}
+static inline void i915_gem_context_put_unlocked(struct i915_gem_context *ctx)
+{
+ struct mutex *lock = &ctx->i915->drm.struct_mutex;
+
+ if (kref_put_mutex(&ctx->ref, i915_gem_context_free, lock))
+ mutex_unlock(lock);
+}
+
static inline struct intel_timeline *
i915_gem_context_lookup_timeline(struct i915_gem_context *ctx,
struct intel_engine_cs *engine)
return &vm->timeline.engine[engine->id];
}
-static inline bool i915_gem_context_is_default(const struct i915_gem_context *c)
-{
- return c->user_handle == DEFAULT_CONTEXT_HANDLE;
-}
-
-int i915_gem_context_create_ioctl(struct drm_device *dev, void *data,
- struct drm_file *file);
-int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data,
- struct drm_file *file);
-int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data,
- struct drm_file *file_priv);
-int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data,
- struct drm_file *file_priv);
-int i915_gem_context_reset_stats_ioctl(struct drm_device *dev, void *data,
- struct drm_file *file);
+int i915_perf_open_ioctl(struct drm_device *dev, void *data,
+ struct drm_file *file);
/* i915_gem_evict.c */
int __must_check i915_gem_evict_something(struct i915_address_space *vm,
unsigned cache_level,
u64 start, u64 end,
unsigned flags);
-int __must_check i915_gem_evict_for_vma(struct i915_vma *target);
+int __must_check i915_gem_evict_for_node(struct i915_address_space *vm,
+ struct drm_mm_node *node,
+ unsigned int flags);
int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle);
/* belongs in i915_gem_gtt.h */
int i915_gem_init_stolen(struct drm_i915_private *dev_priv);
void i915_gem_cleanup_stolen(struct drm_device *dev);
struct drm_i915_gem_object *
-i915_gem_object_create_stolen(struct drm_device *dev, u32 size);
+i915_gem_object_create_stolen(struct drm_i915_private *dev_priv, u32 size);
struct drm_i915_gem_object *
-i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev,
+i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv,
u32 stolen_offset,
u32 gtt_offset,
u32 size);
/* i915_gem_internal.c */
struct drm_i915_gem_object *
i915_gem_object_create_internal(struct drm_i915_private *dev_priv,
- unsigned int size);
+ phys_addr_t size);
/* i915_gem_shrinker.c */
unsigned long i915_gem_shrink(struct drm_i915_private *dev_priv,
i915_gem_object_is_tiled(obj);
}
+u32 i915_gem_fence_size(struct drm_i915_private *dev_priv, u32 size,
+ unsigned int tiling, unsigned int stride);
+u32 i915_gem_fence_alignment(struct drm_i915_private *dev_priv, u32 size,
+ unsigned int tiling, unsigned int stride);
+
/* i915_debugfs.c */
#ifdef CONFIG_DEBUG_FS
int i915_debugfs_register(struct drm_i915_private *dev_priv);
-void i915_debugfs_unregister(struct drm_i915_private *dev_priv);
int i915_debugfs_connector_add(struct drm_connector *connector);
void intel_display_crc_init(struct drm_i915_private *dev_priv);
#else
static inline int i915_debugfs_register(struct drm_i915_private *dev_priv) {return 0;}
-static inline void i915_debugfs_unregister(struct drm_i915_private *dev_priv) {}
static inline int i915_debugfs_connector_add(struct drm_connector *connector)
{ return 0; }
static inline void intel_display_crc_init(struct drm_i915_private *dev_priv) {}
__printf(2, 3)
void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...);
int i915_error_state_to_str(struct drm_i915_error_state_buf *estr,
- const struct i915_error_state_file_priv *error);
+ const struct i915_gpu_state *gpu);
int i915_error_state_buf_init(struct drm_i915_error_state_buf *eb,
struct drm_i915_private *i915,
size_t count, loff_t pos);
{
kfree(eb->buf);
}
+
+struct i915_gpu_state *i915_capture_gpu_state(struct drm_i915_private *i915);
void i915_capture_error_state(struct drm_i915_private *dev_priv,
u32 engine_mask,
const char *error_msg);
-void i915_error_state_get(struct drm_device *dev,
- struct i915_error_state_file_priv *error_priv);
-void i915_error_state_put(struct i915_error_state_file_priv *error_priv);
-void i915_destroy_error_state(struct drm_device *dev);
+
+static inline struct i915_gpu_state *
+i915_gpu_state_get(struct i915_gpu_state *gpu)
+{
+ kref_get(&gpu->ref);
+ return gpu;
+}
+
+void __i915_gpu_state_free(struct kref *kref);
+static inline void i915_gpu_state_put(struct i915_gpu_state *gpu)
+{
+ if (gpu)
+ kref_put(&gpu->ref, __i915_gpu_state_free);
+}
+
+struct i915_gpu_state *i915_first_error_state(struct drm_i915_private *i915);
+void i915_reset_error_state(struct drm_i915_private *i915);
#else
{
}
-static inline void i915_destroy_error_state(struct drm_device *dev)
+static inline struct i915_gpu_state *
+i915_first_error_state(struct drm_i915_private *i915)
+{
+ return NULL;
+}
+
+static inline void i915_reset_error_state(struct drm_i915_private *i915)
{
}
int i915_cmd_parser_get_version(struct drm_i915_private *dev_priv);
void intel_engine_init_cmd_parser(struct intel_engine_cs *engine);
void intel_engine_cleanup_cmd_parser(struct intel_engine_cs *engine);
-bool intel_engine_needs_cmd_parser(struct intel_engine_cs *engine);
int intel_engine_cmd_parser(struct intel_engine_cs *engine,
struct drm_i915_gem_object *batch_obj,
struct drm_i915_gem_object *shadow_batch_obj,
u32 batch_len,
bool is_master);
+/* i915_perf.c */
+extern void i915_perf_init(struct drm_i915_private *dev_priv);
+extern void i915_perf_fini(struct drm_i915_private *dev_priv);
+extern void i915_perf_register(struct drm_i915_private *dev_priv);
+extern void i915_perf_unregister(struct drm_i915_private *dev_priv);
+
/* i915_suspend.c */
-extern int i915_save_state(struct drm_device *dev);
-extern int i915_restore_state(struct drm_device *dev);
+extern int i915_save_state(struct drm_i915_private *dev_priv);
+extern int i915_restore_state(struct drm_i915_private *dev_priv);
/* i915_sysfs.c */
void i915_setup_sysfs(struct drm_i915_private *dev_priv);
void i915_teardown_sysfs(struct drm_i915_private *dev_priv);
+/* intel_lpe_audio.c */
+int intel_lpe_audio_init(struct drm_i915_private *dev_priv);
+void intel_lpe_audio_teardown(struct drm_i915_private *dev_priv);
+void intel_lpe_audio_irq_handler(struct drm_i915_private *dev_priv);
+void intel_lpe_audio_notify(struct drm_i915_private *dev_priv,
+ void *eld, int port, int pipe, int tmds_clk_speed,
+ bool dp_output, int link_rate);
+
/* intel_i2c.c */
-extern int intel_setup_gmbus(struct drm_device *dev);
-extern void intel_teardown_gmbus(struct drm_device *dev);
+extern int intel_setup_gmbus(struct drm_i915_private *dev_priv);
+extern void intel_teardown_gmbus(struct drm_i915_private *dev_priv);
extern bool intel_gmbus_is_valid_pin(struct drm_i915_private *dev_priv,
unsigned int pin);
{
return container_of(adapter, struct intel_gmbus, adapter)->force_bit;
}
-extern void intel_i2c_reset(struct drm_device *dev);
+extern void intel_i2c_reset(struct drm_i915_private *dev_priv);
/* intel_bios.c */
int intel_bios_init(struct drm_i915_private *dev_priv);
return (struct intel_device_info *)&dev_priv->info;
}
+const char *intel_platform_name(enum intel_platform platform);
void intel_device_info_runtime_init(struct drm_i915_private *dev_priv);
void intel_device_info_dump(struct drm_i915_private *dev_priv);
extern void i915_redisable_vga(struct drm_i915_private *dev_priv);
extern void i915_redisable_vga_power_on(struct drm_i915_private *dev_priv);
extern bool ironlake_set_drps(struct drm_i915_private *dev_priv, u8 val);
-extern void intel_init_pch_refclk(struct drm_device *dev);
-extern void intel_set_rps(struct drm_i915_private *dev_priv, u8 val);
-extern void intel_set_memory_cxsr(struct drm_i915_private *dev_priv,
+extern void intel_init_pch_refclk(struct drm_i915_private *dev_priv);
+extern int intel_set_rps(struct drm_i915_private *dev_priv, u8 val);
+extern bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv,
bool enable);
int i915_reg_read_ioctl(struct drm_device *dev, void *data,
extern struct intel_display_error_state *
intel_display_capture_error_state(struct drm_i915_private *dev_priv);
extern void intel_display_print_error_state(struct drm_i915_error_state_buf *e,
- struct drm_i915_private *dev_priv,
struct intel_display_error_state *error);
int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val);
/* intel_sideband.c */
u32 vlv_punit_read(struct drm_i915_private *dev_priv, u32 addr);
-void vlv_punit_write(struct drm_i915_private *dev_priv, u32 addr, u32 val);
+int vlv_punit_write(struct drm_i915_private *dev_priv, u32 addr, u32 val);
u32 vlv_nc_read(struct drm_i915_private *dev_priv, u8 addr);
u32 vlv_iosf_sb_read(struct drm_i915_private *dev_priv, u8 port, u32 reg);
void vlv_iosf_sb_write(struct drm_i915_private *dev_priv, u8 port, u32 reg, u32 val);
void vlv_flisdsi_write(struct drm_i915_private *dev_priv, u32 reg, u32 val);
/* intel_dpio_phy.c */
-void bxt_port_to_phy_channel(enum port port,
+void bxt_port_to_phy_channel(struct drm_i915_private *dev_priv, enum port port,
enum dpio_phy *phy, enum dpio_channel *ch);
void bxt_ddi_phy_set_signal_level(struct drm_i915_private *dev_priv,
enum port port, u32 margin, u32 scale,
}
static inline bool
-__i915_request_irq_complete(struct drm_i915_gem_request *req)
+__i915_request_irq_complete(const struct drm_i915_gem_request *req)
{
struct intel_engine_cs *engine = req->engine;
+ u32 seqno;
+
+ /* Note that the engine may have wrapped around the seqno, and
+ * so our request->global_seqno will be ahead of the hardware,
+ * even though it completed the request before wrapping. We catch
+ * this by kicking all the waiters before resetting the seqno
+ * in hardware, and also signal the fence.
+ */
+ if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &req->fence.flags))
+ return true;
+
+ /* The request was dequeued before we were awoken. We check after
+ * inspecting the hw to confirm that this was the same request
+ * that generated the HWS update. The memory barriers within
+ * the request execution are sufficient to ensure that a check
+ * after reading the value from hw matches this request.
+ */
+ seqno = i915_gem_request_global_seqno(req);
+ if (!seqno)
+ return false;
/* Before we do the heavier coherent read of the seqno,
* check the value (hopefully) in the CPU cacheline.
*/
- if (__i915_gem_request_completed(req))
+ if (__i915_gem_request_completed(req, seqno))
return true;
/* Ensure our read of the seqno is coherent so that we
* is woken.
*/
if (engine->irq_seqno_barrier &&
- rcu_access_pointer(engine->breadcrumbs.irq_seqno_bh) == current &&
- cmpxchg_relaxed(&engine->breadcrumbs.irq_posted, 1, 0)) {
- struct task_struct *tsk;
+ test_and_clear_bit(ENGINE_IRQ_BREADCRUMB, &engine->irq_posted)) {
+ struct intel_breadcrumbs *b = &engine->breadcrumbs;
/* The ordering of irq_posted versus applying the barrier
* is crucial. The clearing of the current irq_posted must
* the seqno before we believe it coherent since they see
* irq_posted == false but we are still running).
*/
- rcu_read_lock();
- tsk = rcu_dereference(engine->breadcrumbs.irq_seqno_bh);
- if (tsk && tsk != current)
+ spin_lock_irq(&b->irq_lock);
+ if (b->irq_wait && b->irq_wait->tsk != current)
/* Note that if the bottom-half is changed as we
* are sending the wake-up, the new bottom-half will
* be woken by whomever made the change. We only have
* to worry about when we steal the irq-posted for
* ourself.
*/
- wake_up_process(tsk);
- rcu_read_unlock();
+ wake_up_process(b->irq_wait->tsk);
+ spin_unlock_irq(&b->irq_lock);
- if (__i915_gem_request_completed(req))
+ if (__i915_gem_request_completed(req, seqno))
return true;
}
void i915_memcpy_init_early(struct drm_i915_private *dev_priv);
bool i915_memcpy_from_wc(void *dst, const void *src, unsigned long len);
+/* The movntdqa instructions used for memcpy-from-wc require 16-byte alignment,
+ * as well as SSE4.1 support. i915_memcpy_from_wc() will report if it cannot
+ * perform the operation. To check beforehand, pass in the parameters to
+ * to i915_can_memcpy_from_wc() - since we only care about the low 4 bits,
+ * you only need to pass in the minor offsets, page-aligned pointers are
+ * always valid.
+ *
+ * For just checking for SSE4.1, in the foreknowledge that the future use
+ * will be correctly aligned, just use i915_has_memcpy_from_wc().
+ */
+#define i915_can_memcpy_from_wc(dst, src, len) \
+ i915_memcpy_from_wc((void *)((unsigned long)(dst) | (unsigned long)(src) | (len)), NULL, 0)
+
+#define i915_has_memcpy_from_wc() \
+ i915_memcpy_from_wc(NULL, NULL, 0)
+
/* i915_mm.c */
int remap_io_mapping(struct vm_area_struct *vma,
unsigned long addr, unsigned long pfn, unsigned long size,
struct io_mapping *iomap);
-#define ptr_mask_bits(ptr) ({ \
- unsigned long __v = (unsigned long)(ptr); \
- (typeof(ptr))(__v & PAGE_MASK); \
-})
-
-#define ptr_unpack_bits(ptr, bits) ({ \
- unsigned long __v = (unsigned long)(ptr); \
- (bits) = __v & ~PAGE_MASK; \
- (typeof(ptr))(__v & PAGE_MASK); \
-})
-
-#define ptr_pack_bits(ptr, bits) \
- ((typeof(ptr))((unsigned long)(ptr) | (bits)))
-
-#define fetch_and_zero(ptr) ({ \
- typeof(*ptr) __T = *(ptr); \
- *(ptr) = (typeof(*ptr))0; \
- __T; \
-})
+static inline bool i915_gem_object_is_coherent(struct drm_i915_gem_object *obj)
+{
+ return (obj->cache_level != I915_CACHE_NONE ||
+ HAS_LLC(to_i915(obj->base.dev)));
+}
#endif
projects / platform / kernel / linux-starfive.git / blobdiff